Merge branch 'feature-Pstream-updates' into 'develop'

Pstream performance and handling improvements

See merge request Development/openfoam!528

applications/solvers/multiphase/icoReactingMultiphaseInterFoam/laserDTRM/Make/options

@@ -3,12 +3,14 @@ phaseSystem = $(LIB_SRC)/phaseSystemModels/multiphaseInter
 EXE_INC = \
     -I${phaseSystem}/phasesSystem/lnInclude \
     -I$(LIB_SRC)/finiteVolume/lnInclude \
+    -I$(LIB_SRC)/fileFormats/lnInclude \
     -I$(LIB_SRC)/meshTools/lnInclude \
     -I$(LIB_SRC)/lagrangian/basic/lnInclude \
     -I$(LIB_SRC)/thermophysicalModels/radiation/lnInclude
 
 LIB_LIBS = \
     -lfiniteVolume \
+    -lfileFormats \
     -lmeshTools \
     -llagrangian \
     -lradiationModels \

applications/solvers/multiphase/icoReactingMultiphaseInterFoam/laserDTRM/laserDTRM.C

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2017-2020 OpenCFD Ltd.
+    Copyright (C) 2017-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -31,11 +31,12 @@ License
 #include "absorptionEmissionModel.H"
 #include "scatterModel.H"
 #include "constants.H"
-#include "addToRunTimeSelectionTable.H"
 #include "unitConversion.H"
 #include "interpolationCell.H"
 #include "interpolationCellPoint.H"
 #include "Random.H"
+#include "OBJstream.H"
+#include "addToRunTimeSelectionTable.H"
 
 using namespace Foam::constant;
 
@@ -679,44 +680,28 @@ void Foam::radiation::laserDTRM::calculate()
         Info<< "Final number of particles..."
             << returnReduce(DTRMCloud_.size(), sumOp<label>()) << endl;
 
-        OFstream osRef(type() + ":particlePath.obj");
+        pointField lines(2*DTRMCloud_.size());
-        label vertI = 0;
+        {
-
+            label i = 0;
-        List<pointField> positions(Pstream::nProcs());
-        List<pointField> p0(Pstream::nProcs());
-
-        DynamicList<point>  positionsMyProc;
-        DynamicList<point>  p0MyProc;
-
             for (const DTRMParticle& p : DTRMCloud_)
             {
-            positionsMyProc.append(p.position());
+                lines[i] = p.position();
-            p0MyProc.append(p.p0());
+                lines[i+1] = p.p0();
+                i += 2;
+            }
         }
 
-        positions[Pstream::myProcNo()].transfer(positionsMyProc);
+        globalIndex::gatherInplaceOp(lines);
-        p0[Pstream::myProcNo()].transfer(p0MyProc);
 
-        Pstream::gatherList(positions);
+        if (Pstream::master())
-        Pstream::scatterList(positions);
-        Pstream::gatherList(p0);
-        Pstream::scatterList(p0);
-
-        for (const int proci : Pstream::allProcs())
         {
-            const pointField& pos = positions[proci];
+            OBJstream os(type() + ":particlePath.obj");
-            const pointField& pfinal = p0[proci];
-            forAll(pos, i)
-            {
-                meshTools::writeOBJ(osRef, pos[i]);
-                vertI++;
-                meshTools::writeOBJ(osRef, pfinal[i]);
-                vertI++;
-                osRef << "l " << vertI-1 << ' ' << vertI << nl;
-            }
-        }
 
-        osRef.flush();
+            for (label pointi = 0; pointi < lines.size(); pointi += 2)
+            {
+                os.write(linePointRef(lines[pointi], lines[pointi+1]));
+            }
+        }
 
         scalar totalQ = gSum(Q_.primitiveFieldRef()*mesh_.V());
         Info << "Total energy absorbed [W]: " << totalQ << endl;

applications/test/parallel-broadcast/Make/files

@@ -0,0 +1,3 @@
+Test-parallel-broadcast.C
+
+EXE = $(FOAM_USER_APPBIN)/Test-parallel-broadcast

applications/test/parallel-broadcast/Make/options

@@ -0,0 +1,2 @@
+/* EXE_INC = */
+/* EXE_LIBS = */

applications/test/parallel-broadcast/Test-parallel-broadcast.C

@@ -0,0 +1,185 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | www.openfoam.com
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+    Copyright (C) 2022 OpenCFD Ltd.
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+Application
+    Test-parallel-broadcast
+
+Description
+    Test for various broadcast routines.
+
+\*---------------------------------------------------------------------------*/
+
+#include "List.H"
+#include "argList.H"
+#include "Time.H"
+#include "bitSet.H"
+#include "vector.H"
+#include "IPstream.H"
+#include "OPstream.H"
+#include "IOstreams.H"
+
+using namespace Foam;
+
+
+template<class T>
+void printPre(const T& value)
+{
+    Info<< nl << "is_contiguous:" << is_contiguous<T>::value << endl;
+    Pout<< "pre-broadcast: " << value << endl;
+}
+
+template<class T>
+void printPost(const T& value)
+{
+    Pout<< "post-broadcast: " << value << endl;
+}
+
+
+template<class T>
+void testBroadcast(T& value)
+{
+    printPre(value);
+    Pstream::broadcast(value);
+    printPost(value);
+}
+
+template<class T>
+void testBroadcast(List<T>& values)
+{
+    Info<< nl << "is_contiguous:" << is_contiguous<T>::value << endl;
+    Pout<< "pre-broadcast: " << flatOutput(values) << endl;
+    Pstream::broadcast(values);
+    Pout<< "post-broadcast: " << flatOutput(values) << endl;
+}
+
+
+void testBroadcast(bitSet& values)
+{
+    Pout<< "pre-broadcast: "
+        << values.size() << ": " << flatOutput(values.values()) << endl;
+    Pstream::broadcast(values);
+    Pout<< "post-broadcast: "
+        << values.size() << ": " << flatOutput(values.values()) << endl;
+}
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+int main(int argc, char *argv[])
+{
+    argList::noCheckProcessorDirectories();
+    #include "setRootCase.H"
+    #include "createTime.H"
+
+    {
+        label value = -1;
+        if (Pstream::master())
+        {
+            value = UPstream::nProcs();
+        }
+        testBroadcast(value);
+    }
+
+    {
+        labelList values;
+        if (Pstream::master())
+        {
+            values = identity(UPstream::nProcs());
+        }
+        testBroadcast(values);
+    }
+
+    {
+        word value;
+        if (Pstream::master())
+        {
+            value = args.executable();
+        }
+        printPre(value);
+        UPstream::broadcast(value);   // Low-level UPstream broadcast
+        printPost(value);
+    }
+
+    {
+        wordList values;
+        if (Pstream::master())
+        {
+            values.resize(UPstream::nProcs());
+            forAll(values, i)
+            {
+                values[i] = "value_" + Foam::name(i);
+            }
+        }
+        testBroadcast(values);
+    }
+
+    {
+        vector values(vector::uniform(-1));
+        if (Pstream::master())
+        {
+            values = vector(1,2,3);
+        }
+        testBroadcast(values);
+    }
+
+    {
+        FixedList<vector, 3> values(vector::uniform(-1));
+        if (Pstream::master())
+        {
+            values = vector(1,2,3);
+
+            scalar mult = 1;
+            for (auto& v : values)
+            {
+                v *= mult;
+                mult += 1;
+            }
+        }
+        testBroadcast(values);
+    }
+
+    {
+        bitSet values;
+        if (Pstream::master())
+        {
+            values.set(labelList({1, 4, 8}));
+            values.resize(10);
+        }
+        else
+        {
+            // Just something different
+            values.set(labelList({0, 2}));
+            values.resize(5);
+        }
+        testBroadcast(values);
+    }
+
+    Info<< "End\n" << endl;
+
+    return 0;
+}
+
+
+// ************************************************************************* //

applications/test/parallel-chunks/Make/files

@@ -0,0 +1,3 @@
+Test-parallel-chunks.C
+
+EXE = $(FOAM_USER_APPBIN)/Test-parallel-chunks

applications/test/parallel-chunks/Make/options

@@ -0,0 +1,2 @@
+/* EXE_INC = */
+/* EXE_LIBS = */

applications/test/parallel-chunks/Test-parallel-chunks.C

@@ -0,0 +1,389 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | www.openfoam.com
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+    Copyright (C) 2022 OpenCFD Ltd.
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+Application
+    Test-parallel-chunks
+
+Description
+    Test for sending contiguous data in chunk-wise.
+    Largely mirrors Pstream::exchange or vice versa
+
+\*---------------------------------------------------------------------------*/
+
+#include "List.H"
+#include "argList.H"
+#include "Time.H"
+#include "IPstream.H"
+#include "OPstream.H"
+#include "IOstreams.H"
+
+using namespace Foam;
+
+
+// Looks like Pstream::exchangeBuf
+template<class T>
+void do_exchangeBuf
+(
+    const label sendSize,
+    const char* sendData,
+    const label recvSize,
+    char* recvData,
+    const int tag,
+    const label comm,
+    const bool wait
+)
+{
+    const label startOfRequests = Pstream::nRequests();
+
+    // Set up receives
+    // ~~~~~~~~~~~~~~~
+
+    // forAll(recvSizes, proci)
+    {
+        // if (proci != Pstream::myProcNo(comm) && recvSizes[proci] > 0)
+        if (!Pstream::master(comm) && recvSize > 0)
+        {
+            UIPstream::read
+            (
+                UPstream::commsTypes::nonBlocking,
+                UPstream::myProcNo(comm),   // proci,
+                recvData,
+                recvSize*sizeof(T),
+                tag,
+                comm
+            );
+        }
+    }
+
+
+    // Set up sends
+    // ~~~~~~~~~~~~
+
+    // forAll(sendBufs, proci)
+    for (const int proci : Pstream::subProcs(comm))
+    {
+        if (sendSize > 0)
+        // if (proci != Pstream::myProcNo(comm) && sendSizes[proci] > 0)
+        {
+            if
+            (
+               !UOPstream::write
+                (
+                    UPstream::commsTypes::nonBlocking,
+                    proci,
+                    sendData,
+                    sendSize*sizeof(T),
+                    tag,
+                    comm
+                )
+            )
+            {
+                FatalErrorInFunction
+                    << "Cannot send outgoing message. "
+                    << "to:" << proci << " nBytes:"
+                    << label(sendSize*sizeof(T))
+                    << Foam::abort(FatalError);
+            }
+        }
+    }
+
+
+    // Wait for all to finish
+    // ~~~~~~~~~~~~~~~~~~~~~~
+
+    if (wait)
+    {
+        UPstream::waitRequests(startOfRequests);
+    }
+}
+
+
+// Looks like Pstream::exchangeContainer
+template<class Container, class T>
+void do_exchangeContainer
+(
+    const Container& sendData,
+    const label recvSize,
+    Container& recvData,
+    const int tag,
+    const label comm,
+    const bool wait
+)
+{
+    const label startOfRequests = Pstream::nRequests();
+
+    // Set up receives
+    // ~~~~~~~~~~~~~~~
+
+    // for (const int proci : Pstream::allProcs(comm))
+    {
+        if (!Pstream::master(comm) && recvSize > 0)
+        // if (proci != Pstream::myProcNo(comm) && recvSize > 0)
+        {
+            UIPstream::read
+            (
+                UPstream::commsTypes::nonBlocking,
+                UPstream::myProcNo(comm),  // proci,
+                recvData.data_bytes(),
+                recvSize*sizeof(T),
+                tag,
+                comm
+            );
+        }
+    }
+
+
+    // Set up sends
+    // ~~~~~~~~~~~~
+
+    if (Pstream::master(comm) && sendData.size() > 0)
+    {
+        for (const int proci : Pstream::subProcs(comm))
+        {
+            if
+            (
+               !UOPstream::write
+                (
+                    UPstream::commsTypes::nonBlocking,
+                    proci,
+                    sendData.cdata_bytes(),
+                    sendData.size_bytes(),
+                    tag,
+                    comm
+                )
+            )
+            {
+                FatalErrorInFunction
+                    << "Cannot send outgoing message. "
+                    << "to:" << proci << " nBytes:"
+                    << label(sendData.size_bytes())
+                    << Foam::abort(FatalError);
+            }
+        }
+    }
+
+    // Wait for all to finish
+    // ~~~~~~~~~~~~~~~~~~~~~~
+
+    if (wait)
+    {
+        UPstream::waitRequests(startOfRequests);
+    }
+}
+
+
+template<class Container, class T>
+void broadcast_chunks
+(
+    Container& sendData,
+    const int tag = UPstream::msgType(),
+    const label comm = UPstream::worldComm,
+    const bool wait = true
+)
+{
+    // OR  static_assert(is_contiguous<T>::value, "Contiguous data only!")
+    if (!is_contiguous<T>::value)
+    {
+        FatalErrorInFunction
+            << "Contiguous data only." << sizeof(T) << Foam::abort(FatalError);
+    }
+
+    if (UPstream::maxCommsSize <= int(sizeof(T)))
+    {
+        // Do in one go
+        Info<< "send " << sendData.size() << " elements in one go" << endl;
+        Pstream::broadcast(sendData, comm);
+        return;
+    }
+
+    label sendSize(sendData.size());
+    Pstream::broadcast(sendSize, comm);
+
+    label recvSize(sendSize);
+
+    sendData.resize_nocopy(recvSize);  // A no-op on master
+
+    // Determine the number of chunks to send. Note that we
+    // only have to look at the sending data since we are
+    // guaranteed that some processor's sending size is some other
+    // processor's receive size. Also we can ignore any local comms.
+
+    // We need to send bytes so the number of iterations:
+    //  maxChunkSize                        iterations
+    //  ------------                        ----------
+    //  0                                   0
+    //  1..maxChunkSize                     1
+    //  maxChunkSize+1..2*maxChunkSize      2
+    //      etc.
+
+    const label maxChunkSize(UPstream::maxCommsSize/sizeof(T));
+
+    label nIter(0);
+    {
+        label nSendMax = 0;
+        // forAll(sendBufs, proci)
+        // {
+        //     if (proci != Pstream::myProcNo(comm))
+        //     {
+        //         nSendMax = max(nSendMax, sendBufs[proci].size());
+        //     }
+        // }
+        nSendMax = sendSize;
+
+        if (nSendMax)
+        {
+            nIter = 1 + ((nSendMax-1)/maxChunkSize);
+        }
+        reduce(nIter, maxOp<label>(), tag, comm);
+
+        Info
+            << "send " << nSendMax << " elements ("
+            << (nSendMax*sizeof(T)) << " bytes) in " << nIter
+            << " iterations of " << maxChunkSize << " chunks ("
+            << (maxChunkSize*sizeof(T)) << " bytes) for maxCommsSize:"
+            << Pstream::maxCommsSize
+            << endl;
+    }
+
+    label nSend(0);
+    label startSend(0);
+    char* charPtrSend;
+
+    for (label iter = 0; iter < nIter; ++iter)
+    {
+        nSend = min
+        (
+            maxChunkSize,
+            sendSize-startSend
+        );
+
+        charPtrSend =
+        (
+            nSend > 0
+          ? reinterpret_cast<char*>(&(sendData[startSend]))
+          : nullptr
+        );
+
+        Info<< "iter " << iter
+            << ": beg=" << startSend << " len=" << nSend
+            << " (" << (nSend*sizeof(T)) << " bytes)" << endl;
+
+        UPstream::broadcast(charPtrSend, nSend*sizeof(T), comm);
+
+        // forAll(nSend, proci)
+        {
+            startSend += nSend;
+        }
+    }
+}
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+int main(int argc, char *argv[])
+{
+    argList::noCheckProcessorDirectories();
+    #include "setRootCase.H"
+    #include "createTime.H"
+
+    if (!Pstream::parRun())
+    {
+        Info<< "\nWarning: not parallel - skipping further tests\n" << endl;
+        return 0;
+    }
+
+    labelList input1;
+    if (Pstream::master())
+    {
+        input1 = identity(500);
+    }
+    broadcast_chunks<labelList, label>(input1);
+
+    Pstream::maxCommsSize = 33;
+    broadcast_chunks<labelList, label>(input1);
+
+    // Mostly the same with PstreamBuffers
+    if (false)
+    {
+        PstreamBuffers pBufs(UPstream::commsTypes::nonBlocking);
+
+        labelList sendData;
+        if (Pstream::master())
+        {
+            sendData = identity(500);
+
+            for (const int proci : Pstream::allProcs())
+            {
+                if (proci != Pstream::myProcNo())
+                {
+                    UOPstream os(proci, pBufs);
+                    os << sendData;
+                }
+            }
+        }
+
+        Info<< "call finishedSends()" << endl;
+        pBufs.finishedSends();
+
+        if (!Pstream::master())
+        {
+            UIPstream is(UPstream::masterNo(), pBufs);
+            is >> sendData;
+        }
+    }
+
+    // Manually
+    {
+        labelListList sendBufs(UPstream::nProcs());
+        labelListList recvBufs(UPstream::nProcs());
+        labelList recvSizes;
+
+        if (Pstream::master())
+        {
+            for (const int proci : Pstream::allProcs())
+            {
+                if (proci != Pstream::myProcNo())
+                {
+                    sendBufs[proci] = identity(500);
+                }
+            }
+        }
+
+        Pstream::exchangeSizes(sendBufs, recvSizes);
+
+        Pstream::exchange<labelList, label>
+        (
+            sendBufs,
+            recvSizes,
+            recvBufs
+        );
+    }
+
+    Info<< "End\n" << endl;
+    return 0;
+}
+
+
+// ************************************************************************* //

applications/test/parallel-communicators/Test-parallel-communicators.C

@@ -163,16 +163,6 @@ int main(int argc, char *argv[])
 
     if (Pstream::myProcNo(comm) != -1)
     {
-        //scalar sum = sumReduce(comm, localValue);
-        //scalar sum = localValue;
-        //reduce
-        //(
-        //    UPstream::treeCommunication(comm),
-        //    sum,
-        //    sumOp<scalar>(),
-        //    Pstream::msgType(),
-        //    comm
-        //);
         scalar sum = returnReduce
         (
             localValue,

applications/test/processorTopology/Make/files

@@ -0,0 +1,3 @@
+Test-processorTopology.C
+
+EXE = $(FOAM_USER_APPBIN)/Test-processorTopology

applications/test/processorTopology/Make/options

@@ -0,0 +1,2 @@
+/* EXE_INC = */
+/* EXE_LIBS = */

applications/test/processorTopology/Test-processorTopology.C

@@ -0,0 +1,113 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | www.openfoam.com
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+    Copyright (C) 2022 OpenCFD Ltd.
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+Description
+    Test/output processor topology
+
+\*---------------------------------------------------------------------------*/
+
+#include "argList.H"
+#include "Time.H"
+#include "polyMesh.H"
+#include "globalMeshData.H"
+#include "OFstream.H"
+
+using namespace Foam;
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+// Main program:
+
+int main(int argc, char *argv[])
+{
+    argList::noFunctionObjects();
+    argList::addNote
+    (
+        "Create a metis graph file representation for an OpenFOAM mesh"
+    );
+
+    #include "setRootCase.H"
+
+    if (!Pstream::parRun())
+    {
+        FatalErrorInFunction
+            << "Only meaningful in parallel"
+            << exit(FatalError);
+    }
+
+    #include "createTime.H"
+    #include "createPolyMesh.H"
+
+    const globalMeshData& globData = mesh.globalData();
+
+    const labelListList& connectivity = globData;
+
+    if (Pstream::master())
+    {
+        OFstream os("processorTopology.dot");
+
+        os << "// processorTopology" << nl << nl;
+        os.beginBlock("graph");
+
+        forAll(connectivity, proci)
+        {
+            label nconn = 0;
+
+            for (const label neighProci : connectivity[proci])
+            {
+                if (proci < neighProci)
+                {
+                    if (nconn++)
+                    {
+                        os << "  ";
+                    }
+                    else
+                    {
+                        os << indent;
+                    }
+                    os << proci << " -- " << neighProci;
+                }
+            }
+
+            if (nconn)
+            {
+                os  << nl;
+            }
+        }
+
+        os.endBlock();
+
+        Info<< "Wrote processorTopology graph: "
+            << runTime.relativePath(os.name()) << nl;
+
+        Info<< nl
+            << "Use neato, circo or fdp graphviz tools" << nl;
+    }
+
+    Info<< nl << "End\n" << endl;
+
+    return 0;
+}
+
+// ************************************************************************* //

applications/utilities/mesh/manipulation/checkMesh/checkTools.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2015-2017 OpenFOAM Foundation
-    Copyright (C) 2015-2021 OpenCFD Ltd.
+    Copyright (C) 2015-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -423,8 +423,8 @@ void Foam::mergeAndWrite
 {
     const polyMesh& mesh = refCast<const polyMesh>(set.db());
 
-    pointField mergedPts;
+    labelField mergedIDs(set.sortedToc());
-    labelList mergedIDs;
+    pointField mergedPts(mesh.points(), mergedIDs);
 
     if (Pstream::parRun())
     {
@@ -432,55 +432,12 @@ void Foam::mergeAndWrite
         // (mesh.globalData().mergePoints etc) since this might
         // hide any synchronisation problem
 
-        globalIndex globalNumbering(mesh.nPoints());
+        // Renumber local ids -> global ids
+        globalIndex(mesh.nPoints()).inplaceToGlobal(mergedIDs);
 
-        mergedPts.setSize(returnReduce(set.size(), sumOp<label>()));
+        globalIndex gatherer(mergedIDs.size(), globalIndex::gatherOnly{});
-        mergedIDs.setSize(mergedPts.size());
+        gatherer.gatherInplace(mergedIDs);
-
+        gatherer.gatherInplace(mergedPts);
-        labelList setPointIDs(set.sortedToc());
-
-        // Get renumbered local data
-        pointField myPoints(mesh.points(), setPointIDs);
-        labelList myIDs(globalNumbering.toGlobal(setPointIDs));
-
-        if (Pstream::master())
-        {
-            // Insert master data first
-            label pOffset = 0;
-            SubList<point>(mergedPts, myPoints.size(), pOffset) = myPoints;
-            SubList<label>(mergedIDs, myIDs.size(), pOffset) = myIDs;
-            pOffset += myPoints.size();
-
-            // Receive slave ones
-            for (const int slave : Pstream::subProcs())
-            {
-                IPstream fromSlave(Pstream::commsTypes::scheduled, slave);
-
-                pointField slavePts(fromSlave);
-                labelList slaveIDs(fromSlave);
-
-                SubList<point>(mergedPts, slavePts.size(), pOffset) = slavePts;
-                SubList<label>(mergedIDs, slaveIDs.size(), pOffset) = slaveIDs;
-                pOffset += slaveIDs.size();
-            }
-        }
-        else
-        {
-            // Construct processor stream with estimate of size. Could
-            // be improved.
-            OPstream toMaster
-            (
-                Pstream::commsTypes::scheduled,
-                Pstream::masterNo(),
-                myPoints.byteSize() + myIDs.byteSize()
-            );
-            toMaster << myPoints << myIDs;
-        }
-    }
-    else
-    {
-        mergedIDs = set.sortedToc();
-        mergedPts = pointField(mesh.points(), mergedIDs);
     }
 
 

applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2012-2017 OpenFOAM Foundation
-    Copyright (C) 2015-2021 OpenCFD Ltd.
+    Copyright (C) 2015-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -88,25 +88,10 @@ Foam::autoPtr<Foam::fvMesh> Foam::loadOrCreateMesh
         );
 
         Pstream::parRun(oldParRun);
-
-        // Send patches
-        for (const int slave : Pstream::subProcs())
-        {
-            OPstream toSlave(Pstream::commsTypes::scheduled, slave);
-            toSlave << patchEntries;
-        }
-    }
-    else
-    {
-        // Receive patches
-        IPstream fromMaster
-        (
-            Pstream::commsTypes::scheduled,
-            Pstream::masterNo()
-        );
-        fromMaster >> patchEntries;
     }
 
+    // Broadcast: send patches to all
+    Pstream::scatter(patchEntries);  // == worldComm;
 
 
     // Dummy meshes

applications/utilities/parallelProcessing/redistributePar/parLagrangianRedistributor.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2015 OpenFOAM Foundation
-    Copyright (C) 2015-2018 OpenCFD Ltd.
+    Copyright (C) 2015-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -185,9 +185,8 @@ Foam::parLagrangianRedistributor::redistributeLagrangianPositions
     }
 
 
-    // Start sending. Sets number of bytes transferred
+    // Start sending
-    labelList allNTrans(Pstream::nProcs());
+    pBufs.finishedSends();
-    pBufs.finishedSends(allNTrans);
 
 
     {
@@ -205,18 +204,15 @@ Foam::parLagrangianRedistributor::redistributeLagrangianPositions
             IDLList<passivePositionParticle>()
         );
 
-
         // Retrieve from receive buffers
-        forAll(allNTrans, procI)
+        for (const int proci : pBufs.allProcs())
         {
-            const label nRec = allNTrans[procI];
+            //Pout<< "Receive from processor" << proci << " : "
+            //    << pBufs.hasRecvData(proci) << endl;
 
-            //Pout<< "From processor " << procI << " receiving bytes " << nRec
+            if (pBufs.hasRecvData(proci))
-            //    << endl;
-
-            if (nRec)
             {
-                UIPstream particleStream(procI, pBufs);
+                UIPstream particleStream(proci, pBufs);
 
                 // Receive particles and locate them
                 IDLList<passivePositionParticle> newParticles

applications/utilities/parallelProcessing/redistributePar/redistributePar.C

@@ -261,15 +261,17 @@ boolList haveFacesFile(const fileName& meshPath)
 {
     const fileName facesPath(meshPath/"faces");
     Info<< "Checking for mesh in " << facesPath << nl << endl;
-    boolList haveMesh(Pstream::nProcs(), false);
+    boolList haveMesh
-    haveMesh[Pstream::myProcNo()] = fileHandler().isFile
     (
-        fileHandler().filePath(facesPath)
+        UPstream::listGatherValues<bool>
+        (
+            fileHandler().isFile(fileHandler().filePath(facesPath))
+        )
     );
-    Pstream::gatherList(haveMesh);
-    Pstream::scatterList(haveMesh);
     Info<< "Per processor mesh availability:" << nl
         << "    " << flatOutput(haveMesh) << nl << endl;
+
+    Pstream::broadcast(haveMesh);
     return haveMesh;
 }
 

applications/utilities/preProcessing/setExprFields/setExprFields.C

@@ -208,8 +208,8 @@ bool setField
 
     doCorrectBoundaryConditions(ctrl.correctBCs, output);
 
-    const label numTotal = returnReduce(output.size(), plusOp<label>());
+    const label numTotal = returnReduce(output.size(), sumOp<label>());
-    reduce(numValuesChanged, plusOp<label>());
+    reduce(numValuesChanged, sumOp<label>());
 
     if (numValuesChanged == numTotal)
     {

applications/utilities/surface/surfaceRedistributePar/surfaceRedistributePar.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2016 OpenFOAM Foundation
-    Copyright (C) 2015-2021 OpenCFD Ltd.
+    Copyright (C) 2015-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -63,34 +63,30 @@ void writeProcStats
 {
     // Determine surface bounding boxes, faces, points
     List<treeBoundBox> surfBb(Pstream::nProcs());
-    {
     surfBb[Pstream::myProcNo()] = treeBoundBox(s.points());
     Pstream::gatherList(surfBb);
-        Pstream::scatterList(surfBb);
-    }
 
-    labelList nPoints(Pstream::nProcs());
+    labelList nPoints(UPstream::listGatherValues<label>(s.points().size()));
-    nPoints[Pstream::myProcNo()] = s.points().size();
+    labelList nFaces(UPstream::listGatherValues<label>(s.size()));
-    Pstream::gatherList(nPoints);
-    Pstream::scatterList(nPoints);
-
-    labelList nFaces(Pstream::nProcs());
-    nFaces[Pstream::myProcNo()] = s.size();
-    Pstream::gatherList(nFaces);
-    Pstream::scatterList(nFaces);
 
+    if (Pstream::master())
+    {
         forAll(surfBb, proci)
         {
             Info<< "processor" << proci << nl;
 
             const List<treeBoundBox>& bbs = meshBb[proci];
-        if (bbs.size())
+            forAll(bbs, i)
             {
-            Info<< "\tMesh bounds          : " << bbs[0] << nl;
+                if (!i)
-            for (label i = 1; i < bbs.size(); i++)
                 {
-                Info<< "\t                       " << bbs[i]<< nl;
+                    Info<< "\tMesh bounds          : ";
                 }
+                else
+                {
+                    Info<< "\t                       ";
+                }
+                Info<< bbs[i] << nl;
             }
             Info<< "\tSurface bounding box : " << surfBb[proci] << nl
                 << "\tTriangles            : " << nFaces[proci] << nl
@@ -99,6 +95,7 @@ void writeProcStats
         }
         Info<< endl;
     }
+}
 
 
 

src/OpenFOAM/Make/files

@@ -266,14 +266,16 @@ StringStreams = $(Streams)/StringStreams
 $(StringStreams)/StringStream.C
 
 Pstreams = $(Streams)/Pstreams
-$(Pstreams)/UIPstream.C
+/* $(Pstreams)/UPstream.C in global.C */
-$(Pstreams)/IPstream.C
-/* $(Pstreams)/UPstream.C in global.Cver */
 $(Pstreams)/UPstreamCommsStruct.C
 $(Pstreams)/Pstream.C
-$(Pstreams)/UOPstream.C
-$(Pstreams)/OPstream.C
 $(Pstreams)/PstreamBuffers.C
+$(Pstreams)/UIPstreamBase.C
+$(Pstreams)/UOPstreamBase.C
+$(Pstreams)/IPstreams.C
+$(Pstreams)/OPstreams.C
+$(Pstreams)/IPBstreams.C
+$(Pstreams)/OPBstreams.C
 
 dictionary = db/dictionary
 $(dictionary)/dictionary.C
@@ -590,8 +592,6 @@ polyBoundaryMesh = $(polyMesh)/polyBoundaryMesh
 $(polyBoundaryMesh)/polyBoundaryMesh.C
 $(polyBoundaryMesh)/polyBoundaryMeshEntries.C
 
-meshes/ProcessorTopology/commSchedule.C
-
 globalMeshData = $(polyMesh)/globalMeshData
 $(globalMeshData)/globalMeshData.C
 $(globalMeshData)/globalPoints.C
@@ -810,6 +810,7 @@ algorithms/indexedOctree/volumeType.C
 algorithms/dynamicIndexedOctree/dynamicIndexedOctreeName.C
 algorithms/dynamicIndexedOctree/dynamicTreeDataPoint.C
 
+parallel/commSchedule/commSchedule.C
 parallel/globalIndex/globalIndex.C
 
 meshes/data/data.C

src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.C

@@ -394,8 +394,7 @@ bool Foam::decomposedBlockData::readBlocks
             }
         }
 
-        labelList recvSizes;
+        pBufs.finishedSends();
-        pBufs.finishedSends(recvSizes);
 
         if (!UPstream::master(comm))
         {
@@ -524,8 +523,7 @@ Foam::autoPtr<Foam::ISstream> Foam::decomposedBlockData::readBlocks
             ok = is.good();
         }
 
-        labelList recvSizes;
+        pBufs.finishedSends();
-        pBufs.finishedSends(recvSizes);
 
         if (!UPstream::master(comm))
         {

src/OpenFOAM/db/IOstreams/Fstreams/masterOFstream.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2017 OpenFOAM Foundation
-    Copyright (C) 2020-2021 OpenCFD Ltd.
+    Copyright (C) 2020-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -115,10 +115,7 @@ void Foam::masterOFstream::commit()
             return;
         }
 
-        boolList valid(Pstream::nProcs());
+        boolList valid(UPstream::listGatherValues<bool>(valid_));
-        valid[Pstream::myProcNo()] = valid_;
-        Pstream::gatherList(valid);
-
 
         // Different files
         PstreamBuffers pBufs(Pstream::commsTypes::nonBlocking);

src/OpenFOAM/db/IOstreams/Pstreams/IPBstreams.C

@@ -5,8 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2011-2015 OpenFOAM Foundation
+    Copyright (C) 2022 OpenCFD Ltd.
-    Copyright (C) 2021 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -24,16 +23,15 @@ License
     You should have received a copy of the GNU General Public License
     along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
 
-Description
-    Read from UIPstream
-
 \*---------------------------------------------------------------------------*/
 
 #include "UIPstream.H"
+#include "IPstream.H"
+#include "IOstreams.H"
 
 // * * * * * * * * * * * * * * * * Constructor * * * * * * * * * * * * * * * //
 
-Foam::UIPstream::UIPstream
+Foam::UIPBstream::UIPBstream
 (
     const commsTypes commsType,
     const int fromProcNo,
@@ -45,51 +43,65 @@ Foam::UIPstream::UIPstream
     IOstreamOption::streamFormat fmt
 )
 :
-    UPstream(commsType),
+    UIPstreamBase
-    Istream(fmt, IOstreamOption::currentVersion),
+    (
-    fromProcNo_(fromProcNo),
+        commsType,
-    recvBuf_(receiveBuf),
+        fromProcNo,
-    recvBufPos_(receiveBufPosition),
+        receiveBuf,
-    tag_(tag),
+        receiveBufPosition,
-    comm_(comm),
+        tag,
-    clearAtEnd_(clearAtEnd),
+        comm,
-    messageSize_(0)
+        clearAtEnd,
+        fmt
+    )
 {
-    NotImplemented;
+    bufferIPCrecv();
 }
 
 
-Foam::UIPstream::UIPstream(const int fromProcNo, PstreamBuffers& buffers)
+Foam::IPBstream::IPBstream
-:
-    UPstream(buffers.commsType_),
-    Istream(buffers.format_, IOstreamOption::currentVersion),
-    fromProcNo_(fromProcNo),
-    recvBuf_(buffers.recvBuf_[fromProcNo]),
-    recvBufPos_(buffers.recvBufPos_[fromProcNo]),
-    tag_(buffers.tag_),
-    comm_(buffers.comm_),
-    clearAtEnd_(true),
-    messageSize_(0)
-{
-    NotImplemented;
-}
-
-
-// * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
-
-Foam::label Foam::UIPstream::read
 (
     const commsTypes commsType,
     const int fromProcNo,
-    char* buf,
+    const label bufSize,
-    const std::streamsize bufSize,
     const int tag,
-    const label communicator
+    const label comm,
+    IOstreamOption::streamFormat fmt
 )
-{
+:
-    NotImplemented;
+    Pstream(commsType, bufSize),
-    return 0;
+    UIPBstream
-}
+    (
+        commsType,
+        fromProcNo,
+        Pstream::transferBuf_,
+        transferBufPosition_,
+        tag,
+        comm,
+        false,  // Do not clear Pstream::transferBuf_ if at end
+        fmt
+    ),
+    transferBufPosition_(0)
+{}
+
+
+Foam::IPBstream::IPBstream
+(
+    const int fromProcNo,
+    const label comm,
+    IOstreamOption::streamFormat fmt
+)
+:
+    IPBstream
+    (
+        UPstream::commsTypes::scheduled,    // irrelevant
+        fromProcNo,
+        label(0),  // bufSize
+        UPstream::msgType(),                // irrelevant
+        comm,
+        fmt
+    )
+{}
 
 
 // ************************************************************************* //

src/OpenFOAM/db/IOstreams/Pstreams/IPstream.H

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2013 OpenFOAM Foundation
-    Copyright (C) 2021 OpenCFD Ltd.
+    Copyright (C) 2021-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -31,14 +31,14 @@ Description
     Input inter-processor communications stream.
 
 SourceFiles
-    IPstream.C
+    IPstreams.C
 
 \*---------------------------------------------------------------------------*/
 
 #include "Pstream.H"
 
-#ifndef IPstream_H
+#ifndef Foam_IPstream_H
-#define IPstream_H
+#define Foam_IPstream_H
 
 #include "UIPstream.H"
 
@@ -79,6 +79,48 @@ public:
 };
 
 
+/*---------------------------------------------------------------------------*\
+                          Class IPBstream Declaration
+\*---------------------------------------------------------------------------*/
+
+//- Input inter-processor communications stream
+//- using MPI broadcast.
+class IPBstream
+:
+    public Pstream,
+    public UIPBstream
+{
+    // Private Data
+
+        //- Receive index into Pstream::transferBuf_
+        label transferBufPosition_;
+
+public:
+
+    // Constructors
+
+        //- Construct for broadcast root, optional buffer size, read format
+        IPBstream
+        (
+            const commsTypes commsType,     //!< ignored
+            const int rootProcNo,           //!< normally UPstream::masterNo()
+            const label bufSize = 0,
+            const int tag = UPstream::msgType(),  //!< ignored
+            const label comm = UPstream::worldComm,
+            IOstreamOption::streamFormat fmt = IOstreamOption::BINARY
+        );
+
+        //- Construct for broadcast root with optional communicator,
+        //- write format
+        explicit IPBstream
+        (
+            const int rootProcNo,           //!< normally UPstream::masterNo()
+            const label comm = UPstream::worldComm,
+            IOstreamOption::streamFormat fmt = IOstreamOption::BINARY
+        );
+};
+
+
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
 } // End namespace Foam

src/OpenFOAM/db/IOstreams/Pstreams/IPstreams.C

@@ -0,0 +1,120 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | www.openfoam.com
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+    Copyright (C) 2022 OpenCFD Ltd.
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "UIPstream.H"
+#include "IPstream.H"
+#include "IOstreams.H"
+
+// * * * * * * * * * * * * * * * * Constructor * * * * * * * * * * * * * * * //
+
+Foam::UIPstream::UIPstream
+(
+    const commsTypes commsType,
+    const int fromProcNo,
+    DynamicList<char>& receiveBuf,
+    label& receiveBufPosition,
+    const int tag,
+    const label comm,
+    const bool clearAtEnd,
+    IOstreamOption::streamFormat fmt
+)
+:
+    UIPstreamBase
+    (
+        commsType,
+        fromProcNo,
+        receiveBuf,
+        receiveBufPosition,
+        tag,
+        comm,
+        clearAtEnd,
+        fmt
+    )
+{
+    if (commsType == commsTypes::nonBlocking)
+    {
+        // Message is already received into buffer
+    }
+    else
+    {
+        bufferIPCrecv();
+    }
+}
+
+
+Foam::UIPstream::UIPstream(const int fromProcNo, PstreamBuffers& buffers)
+:
+    UIPstreamBase(fromProcNo, buffers)
+{
+    if (commsType() == commsTypes::nonBlocking)
+    {
+        // Message is already received into buffer
+        messageSize_ = recvBuf_.size();
+
+        if (debug)
+        {
+            Pout<< "UIPstream::UIPstream PstreamBuffers :"
+                << " fromProcNo:" << fromProcNo_
+                << " tag:" << tag_ << " comm:" << comm_
+                << " receive buffer size:" << messageSize_
+                << Foam::endl;
+        }
+    }
+    else
+    {
+        bufferIPCrecv();
+    }
+}
+
+
+Foam::IPstream::IPstream
+(
+    const commsTypes commsType,
+    const int fromProcNo,
+    const label bufSize,
+    const int tag,
+    const label comm,
+    IOstreamOption::streamFormat fmt
+)
+:
+    Pstream(commsType, bufSize),
+    UIPstream
+    (
+        commsType,
+        fromProcNo,
+        Pstream::transferBuf_,
+        transferBufPosition_,
+        tag,
+        comm,
+        false,  // Do not clear Pstream::transferBuf_ if at end
+        fmt
+    ),
+    transferBufPosition_(0)
+{}
+
+
+// ************************************************************************* //

src/OpenFOAM/db/IOstreams/Pstreams/OPBstreams.C

@@ -0,0 +1,109 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | www.openfoam.com
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+    Copyright (C) 2022 OpenCFD Ltd.
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "UOPstream.H"
+#include "OPstream.H"
+#include "IOstreams.H"
+
+// * * * * * * * * * * * * * * * * Constructor * * * * * * * * * * * * * * * //
+
+Foam::UOPBstream::UOPBstream
+(
+    const commsTypes commsType,
+    const int toProcNo,
+    DynamicList<char>& sendBuf,
+    const int tag,
+    const label comm,
+    const bool sendAtDestruct,
+    IOstreamOption::streamFormat fmt
+)
+:
+    UOPstreamBase(commsType, toProcNo, sendBuf, tag, comm, sendAtDestruct, fmt)
+{}
+
+
+Foam::OPBstream::OPBstream
+(
+    const commsTypes commsType,
+    const int toProcNo,
+    const label bufSize,
+    const int tag,
+    const label comm,
+    IOstreamOption::streamFormat fmt
+)
+:
+    Pstream(commsType, bufSize),
+    UOPBstream
+    (
+        commsType,
+        toProcNo,
+        Pstream::transferBuf_,
+        tag,
+        comm,
+        true,  // sendAtDestruct
+        fmt
+    )
+{}
+
+
+Foam::OPBstream::OPBstream
+(
+    const int toProcNo,
+    const label comm,
+    IOstreamOption::streamFormat fmt
+)
+:
+    OPBstream
+    (
+        UPstream::commsTypes::scheduled,    // irrelevant
+        toProcNo,
+        label(0),  // bufSize
+        UPstream::msgType(),                // irrelevant
+        comm,
+        fmt
+    )
+{}
+
+
+// * * * * * * * * * * * * * * * * Destructor  * * * * * * * * * * * * * * * //
+
+Foam::UOPBstream::~UOPBstream()
+{
+    if (sendAtDestruct_)
+    {
+        if (!bufferIPCsend())
+        {
+            FatalErrorInFunction
+                << "Failed broadcast message of size "
+                << sendBuf_.size() << " root: " << toProcNo_
+                << Foam::abort(FatalError);
+        }
+    }
+}
+
+
+// ************************************************************************* //

src/OpenFOAM/db/IOstreams/Pstreams/OPstream.H

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2013 OpenFOAM Foundation
-    Copyright (C) 2021 OpenCFD Ltd.
+    Copyright (C) 2021-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -31,14 +31,14 @@ Description
     Output inter-processor communications stream.
 
 SourceFiles
-    OPstream.C
+    OPstreams.C
 
 \*---------------------------------------------------------------------------*/
 
 #include "Pstream.H"
 
-#ifndef OPstream_H
+#ifndef Foam_OPstream_H
-#define OPstream_H
+#define Foam_OPstream_H
 
 #include "UOPstream.H"
 
@@ -74,6 +74,43 @@ public:
 };
 
 
+/*---------------------------------------------------------------------------*\
+                        Class OPBstream Declaration
+\*---------------------------------------------------------------------------*/
+
+//- Output inter-processor communications stream
+//- using MPI broadcast.
+class OPBstream
+:
+    public Pstream,
+    public UOPBstream
+{
+public:
+
+    // Constructors
+
+        //- Construct for broadcast root, optional buffer size, write format
+        OPBstream
+        (
+            const commsTypes commsType,     //!< ignored
+            const int rootProcNo,           //!< normally UPstream::masterNo()
+            const label bufSize = 0,
+            const int tag = UPstream::msgType(),  //!< ignored
+            const label comm = UPstream::worldComm,
+            IOstreamOption::streamFormat fmt = IOstreamOption::BINARY
+        );
+
+        //- Construct for broadcast root with optional communicator,
+        //- write format
+        explicit OPBstream
+        (
+            const int rootProcNo,           //!< normally UPstream::masterNo()
+            const label comm = UPstream::worldComm,
+            IOstreamOption::streamFormat fmt = IOstreamOption::BINARY
+        );
+};
+
+
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
 } // End namespace Foam

src/OpenFOAM/db/IOstreams/Pstreams/OPstreams.C

@@ -0,0 +1,96 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | www.openfoam.com
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+    Copyright (C) 2011 OpenFOAM Foundation
+    Copyright (C) 2022 OpenCFD Ltd.
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "UOPstream.H"
+#include "OPstream.H"
+
+// * * * * * * * * * * * * * * * * Constructor * * * * * * * * * * * * * * * //
+
+Foam::UOPstream::UOPstream
+(
+    const commsTypes commsType,
+    const int toProcNo,
+    DynamicList<char>& sendBuf,
+    const int tag,
+    const label comm,
+    const bool sendAtDestruct,
+    IOstreamOption::streamFormat fmt
+)
+:
+    UOPstreamBase(commsType, toProcNo, sendBuf, tag, comm, sendAtDestruct, fmt)
+{}
+
+
+Foam::UOPstream::UOPstream(const int toProcNo, PstreamBuffers& buffers)
+:
+    UOPstreamBase(toProcNo, buffers)
+{}
+
+
+Foam::OPstream::OPstream
+(
+    const commsTypes commsType,
+    const int toProcNo,
+    const label bufSize,
+    const int tag,
+    const label comm,
+    IOstreamOption::streamFormat fmt
+)
+:
+    Pstream(commsType, bufSize),
+    UOPstream
+    (
+        commsType,
+        toProcNo,
+        Pstream::transferBuf_,
+        tag,
+        comm,
+        true,  // sendAtDestruct
+        fmt
+    )
+{}
+
+
+// * * * * * * * * * * * * * * * * Destructor  * * * * * * * * * * * * * * * //
+
+Foam::UOPstream::~UOPstream()
+{
+    if (sendAtDestruct_)
+    {
+        if (!bufferIPCsend())
+        {
+            FatalErrorInFunction
+                << "Failed sending outgoing message of size "
+                << sendBuf_.size() << " to processor " << toProcNo_
+                << Foam::abort(FatalError);
+        }
+    }
+}
+
+
+// ************************************************************************* //

src/OpenFOAM/db/IOstreams/Pstreams/Pstream.C

@@ -5,7 +5,8 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2011-2012 OpenFOAM Foundation
+    Copyright (C) 2011 OpenFOAM Foundation
+    Copyright (C) 2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -26,6 +27,7 @@ License
 \*---------------------------------------------------------------------------*/
 
 #include "Pstream.H"
+#include "bitSet.H"
 
 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
 
@@ -35,4 +37,40 @@ defineTypeNameAndDebug(Pstream, 0);
 }
 
 
+// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * //
+
+void Foam::Pstream::broadcast
+(
+    bitSet& values,
+    const label comm
+)
+{
+    if (UPstream::parRun() && UPstream::nProcs(comm) > 1)
+    {
+        // Broadcast the size
+        label len(values.size());
+        UPstream::broadcast
+        (
+            reinterpret_cast<char*>(&len),
+            sizeof(label),
+            comm,
+            UPstream::masterNo()
+        );
+
+        values.resize_nocopy(len);  // A no-op on master
+
+        if (len)
+        {
+            UPstream::broadcast
+            (
+                values.data_bytes(),
+                values.size_bytes(),
+                comm,
+                UPstream::masterNo()
+            );
+        }
+    }
+}
+
+
 // ************************************************************************* //

src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2016 OpenFOAM Foundation
-    Copyright (C) 2016-2021 OpenCFD Ltd.
+    Copyright (C) 2016-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -32,24 +32,31 @@ Description
 
 SourceFiles
     Pstream.C
-    combineGatherScatter.C
+    PstreamBroadcast.C
-    gatherScatter.C
+    PstreamGather.C
-    gatherScatterList.C
+    PstreamCombineGather.C
-    exchange.C
+    PstreamGatherList.C
+    PstreamExchange.C
 
 \*---------------------------------------------------------------------------*/
 
-#ifndef Pstream_H
+#ifndef Foam_Pstream_H
-#define Pstream_H
+#define Foam_Pstream_H
 
 #include "UPstream.H"
 #include "DynamicList.H"
 
+// Legacy
+// #define Foam_Pstream_scatter_nobroadcast
+
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
 namespace Foam
 {
 
+// Forward Declarations
+class bitSet;
+
 /*---------------------------------------------------------------------------*\
                            Class Pstream Declaration
 \*---------------------------------------------------------------------------*/
@@ -60,33 +67,31 @@ class Pstream
 {
    // Private Static Functions
 
-        //- Exchange contiguous data. Sends sendData, receives into
+        //- Exchange contiguous data. Sends sendBufs, receives into recvBufs.
-        //  recvData. If block=true will wait for all transfers to finish.
         //  Data provided and received as container.
         template<class Container, class T>
         static void exchangeContainer
         (
             const UList<Container>& sendBufs,
-            const labelUList& recvSizes,
+            const labelUList& recvSizes,  //!< Num of recv elements (not bytes)
             List<Container>& recvBufs,
             const int tag,
             const label comm,
-            const bool block
+            const bool wait               //!< Wait for requests to complete
         );
 
-        //- Exchange contiguous data. Sends sendData, receives into
+        //- Exchange contiguous data. Sends sendBufs, receives into recvBufs.
-        //  recvData. If block=true will wait for all transfers to finish.
         //  Data provided and received as pointers.
         template<class T>
         static void exchangeBuf
         (
-            const labelUList& sendSizes,    // number of T, not number of char
+            const labelUList& sendSizes,  //!< Num of send elements (not bytes)
             const UList<const char*>& sendBufs,
-            const labelUList& recvSizes,    // number of T, not number of char
+            const labelUList& recvSizes,  //!< Num of recv elements (not bytes)
             List<char*>& recvBufs,
             const int tag,
             const label comm,
-            const bool block
+            const bool wait               //!< Wait for requests to complete
         );
 
 
@@ -106,31 +111,82 @@ public:
 
     // Constructors
 
-        //- Construct given optional buffer size
+        //- Construct for given commsTypes, with optional buffer size
         explicit Pstream
         (
-            const commsTypes commsType,
+            const UPstream::commsTypes commsType,
             const label bufSize = 0
         )
         :
             UPstream(commsType)
         {
-            if (bufSize)
+            if (bufSize > 0)
             {
                 transferBuf_.setCapacity(bufSize + 2*sizeof(scalar) + 1);
             }
         }
 
 
-        // Gather and scatter
+    // Static Functions
+
+        // Broadcast
+
+        //- Broadcast buffer or string content to all processes in communicator
+        using UPstream::broadcast;
+
+        //- Generic broadcast using streams to serialize/de-serialize
+        template<class T>
+        static void genericBroadcast
+        (
+            T& value,
+            const label comm = UPstream::worldComm
+        );
+
+        //- Broadcast value (contiguous or non-contiguous)
+        //- to all processes in communicator.
+        template<class T>
+        static void broadcast
+        (
+            T& value,
+            const label comm = UPstream::worldComm
+        );
+
+        //- Broadcast multiple values (contiguous or non-contiguous)
+        //- to all processes in communicator.
+        template<class T>
+        static void broadcast
+        (
+            List<T>& values,
+            const label comm = UPstream::worldComm
+        );
+
+        //- Broadcast multiple values (contiguous or non-contiguous)
+        //- to all processes in communicator.
+        template<class T, int SizeMin>
+        static void broadcast
+        (
+            DynamicList<T, SizeMin>& values,
+            const label comm = UPstream::worldComm
+        );
+
+        //- Broadcast bitSet values
+        //- to all processes in communicator.
+        static void broadcast
+        (
+            bitSet& values,
+            const label comm = UPstream::worldComm
+        );
+
+
+        // Gather
 
             //- Gather data.
-            //- Apply bop to combine Value from different processors
+            //- Apply \c bop to combine \c value from different processors
             template<class T, class BinaryOp>
             static void gather
             (
                 const List<commsStruct>& comms,
-                T& Value,
+                T& value,
                 const BinaryOp& bop,
                 const int tag,
                 const label comm
@@ -140,40 +196,22 @@ public:
             template<class T, class BinaryOp>
             static void gather
             (
-                T& Value,
+                T& value,
                 const BinaryOp& bop,
-                const int tag = Pstream::msgType(),
+                const int tag = UPstream::msgType(),
-                const label comm = Pstream::worldComm
+                const label comm = UPstream::worldComm
-            );
-
-            //- Scatter data. Distribute without modification. Reverse of gather
-            template<class T>
-            static void scatter
-            (
-                const List<commsStruct>& comms,
-                T& Value,
-                const int tag,
-                const label comm
-            );
-
-            //- Like above but switches between linear/tree communication
-            template<class T>
-            static void scatter
-            (
-                T& Value,
-                const int tag = Pstream::msgType(),
-                const label comm = Pstream::worldComm
             );
 
 
-        // Combine variants. Inplace combine values from processors.
+        // Gather/combine data
+        // Inplace combine values from processors.
         // (Uses construct from Istream instead of <<)
 
             template<class T, class CombineOp>
             static void combineGather
             (
                 const List<commsStruct>& comms,
-                T& Value,
+                T& value,
                 const CombineOp& cop,
                 const int tag,
                 const label comm
@@ -183,29 +221,10 @@ public:
             template<class T, class CombineOp>
             static void combineGather
             (
-                T& Value,
+                T& value,
                 const CombineOp& cop,
-                const int tag = Pstream::msgType(),
+                const int tag = UPstream::msgType(),
-                const label comm = Pstream::worldComm
+                const label comm = UPstream::worldComm
-            );
-
-            //- Scatter data. Reverse of combineGather
-            template<class T>
-            static void combineScatter
-            (
-                const List<commsStruct>& comms,
-                T& Value,
-                const int tag,
-                const label comm
-            );
-
-            //- Like above but switches between linear/tree communication
-            template<class T>
-            static void combineScatter
-            (
-                T& Value,
-                const int tag = Pstream::msgType(),
-                const label comm = Pstream::worldComm
             );
 
 
@@ -215,7 +234,7 @@ public:
             static void listCombineGather
             (
                 const List<commsStruct>& comms,
-                List<T>& Value,
+                List<T>& value,
                 const CombineOp& cop,
                 const int tag,
                 const label comm
@@ -225,40 +244,21 @@ public:
             template<class T, class CombineOp>
             static void listCombineGather
             (
-                List<T>& Value,
+                List<T>& values,
                 const CombineOp& cop,
-                const int tag = Pstream::msgType(),
+                const int tag = UPstream::msgType(),
-                const label comm = Pstream::worldComm
+                const label comm = UPstream::worldComm
-            );
-
-            //- Scatter data. Reverse of combineGather
-            template<class T>
-            static void listCombineScatter
-            (
-                const List<commsStruct>& comms,
-                List<T>& Value,
-                const int tag,
-                const label comm
-            );
-
-            //- Like above but switches between linear/tree communication
-            template<class T>
-            static void listCombineScatter
-            (
-                List<T>& Value,
-                const int tag = Pstream::msgType(),
-                const label comm = Pstream::worldComm
             );
 
 
-        // Combine variants working on whole map at a time. Container needs to
+        // Combine variants working on whole map at a time.
-        // have iterators and find() defined.
+        // Container needs iterators, find() and insert methods defined.
 
             template<class Container, class CombineOp>
             static void mapCombineGather
             (
                 const List<commsStruct>& comms,
-                Container& Values,
+                Container& values,
                 const CombineOp& cop,
                 const int tag,
                 const label comm
@@ -268,43 +268,23 @@ public:
             template<class Container, class CombineOp>
             static void mapCombineGather
             (
-                Container& Values,
+                Container& values,
                 const CombineOp& cop,
-                const int tag = Pstream::msgType(),
+                const int tag = UPstream::msgType(),
                 const label comm = UPstream::worldComm
             );
 
-            //- Scatter data. Reverse of combineGather
-            template<class Container>
-            static void mapCombineScatter
-            (
-                const List<commsStruct>& comms,
-                Container& Values,
-                const int tag,
-                const label comm
-            );
-
-            //- Like above but switches between linear/tree communication
-            template<class Container>
-            static void mapCombineScatter
-            (
-                Container& Values,
-                const int tag = Pstream::msgType(),
-                const label comm = UPstream::worldComm
-            );
-
-
 
         // Gather/scatter keeping the individual processor data separate.
-        // Values is a List of size UPstream::nProcs() where
+        // The values is a List of size UPstream::nProcs() where
-        // Values[UPstream::myProcNo()] is the data for the current processor.
+        // values[UPstream::myProcNo()] is the data for the current processor.
 
             //- Gather data but keep individual values separate
             template<class T>
             static void gatherList
             (
                 const List<commsStruct>& comms,
-                List<T>& Values,
+                List<T>& values,
                 const int tag,
                 const label comm
             );
@@ -313,17 +293,109 @@ public:
             template<class T>
             static void gatherList
             (
-                List<T>& Values,
+                List<T>& values,
-                const int tag = Pstream::msgType(),
+                const int tag = UPstream::msgType(),
                 const label comm = UPstream::worldComm
             );
 
+
+        // Scatter
+
+            //- Broadcast data: Distribute without modification.
+            //  \note comms and tag parameters only used when
+            //  Foam_Pstream_scatter_nobroadcast is defined
+            template<class T>
+            static void scatter
+            (
+                const List<commsStruct>& comms,
+                T& value,
+                const int tag,
+                const label comm
+            );
+
+            //- Broadcast data: Distribute without modification.
+            //  \note tag parameter only used when
+            //  Foam_Pstream_scatter_nobroadcast is defined
+            template<class T>
+            static void scatter
+            (
+                T& value,
+                const int tag = UPstream::msgType(),
+                const label comm = UPstream::worldComm
+            );
+
+            //- Broadcast data: Distribute without modification.
+            //  \note tag parameter only used when
+            //  Foam_Pstream_scatter_nobroadcast is defined
+            template<class T>
+            static void combineScatter
+            (
+                const List<commsStruct>& comms,
+                T& value,
+                const int tag,
+                const label comm
+            );
+
+            //- Broadcast data: Distribute without modification.
+            //  \note tag parameter only used when
+            //  Foam_Pstream_scatter_nobroadcast is defined
+            template<class T>
+            static void combineScatter
+            (
+                T& value,
+                const int tag = UPstream::msgType(),
+                const label comm = UPstream::worldComm
+            );
+
+            //- Broadcast data: Distribute without modification.
+            //  \note comms and tag parameters only used when
+            //  Foam_Pstream_scatter_nobroadcast is defined
+            template<class T>
+            static void listCombineScatter
+            (
+                const List<commsStruct>& comms,
+                List<T>& value,
+                const int tag,
+                const label comm
+            );
+
+            //- Broadcast data: Distribute without modification.
+            //  \note comms and tag parameters only used when
+            //  Foam_Pstream_scatter_nobroadcast is defined
+            template<class T>
+            static void listCombineScatter
+            (
+                List<T>& value,
+                const int tag = UPstream::msgType(),
+                const label comm = UPstream::worldComm
+            );
+
+            //- Broadcast data: Distribute without modification.
+            template<class Container>
+            static void mapCombineScatter
+            (
+                const List<commsStruct>& comms,
+                Container& values,
+                const int tag,
+                const label comm
+            );
+
+            //- Like above but switches between linear/tree communication
+            template<class Container>
+            static void mapCombineScatter
+            (
+                Container& values,
+                const int tag = UPstream::msgType(),
+                const label comm = UPstream::worldComm
+            );
+
+
             //- Scatter data. Reverse of gatherList
             template<class T>
             static void scatterList
             (
                 const List<commsStruct>& comms,
-                List<T>& Values,
+                List<T>& values,
                 const int tag,
                 const label comm
             );
@@ -332,25 +404,25 @@ public:
             template<class T>
             static void scatterList
             (
-                List<T>& Values,
+                List<T>& values,
-                const int tag = Pstream::msgType(),
+                const int tag = UPstream::msgType(),
                 const label comm = UPstream::worldComm
             );
 
 
         // Exchange
 
-            //- Helper: exchange contiguous data. Sends sendData, receives into
+            //- Helper: exchange sizes of sendData for specified
-            //  recvData. If block=true will wait for all transfers to finish.
+            //- set of send/receive processes.
-            template<class Container, class T>
+            template<class Container>
-            static void exchange
+            static void exchangeSizes
             (
-                const UList<Container>& sendData,
+                const labelUList& sendProcs,
-                const labelUList& recvSizes,
+                const labelUList& recvProcs,
-                List<Container>& recvData,
+                const Container& sendData,
-                const int tag = UPstream::msgType(),
+                labelList& sizes,
-                const label comm = UPstream::worldComm,
+                const label tag = UPstream::msgType(),
-                const bool block = true
+                const label comm = UPstream::worldComm
             );
 
             //- Helper: exchange sizes of sendData. sendData is the data per
@@ -364,6 +436,20 @@ public:
                 const label comm = UPstream::worldComm
             );
 
+
+            //- Helper: exchange contiguous data. Sends sendData, receives into
+            //  recvData. If block=true will wait for all transfers to finish.
+            template<class Container, class T>
+            static void exchange
+            (
+                const UList<Container>& sendData,
+                const labelUList& recvSizes,
+                List<Container>& recvData,
+                const int tag = UPstream::msgType(),
+                const label comm = UPstream::worldComm,
+                const bool wait = true  //!< Wait for requests to complete
+            );
+
             //- Exchange contiguous data. Sends sendData, receives into
             //  recvData. Determines sizes to receive.
             //  If block=true will wait for all transfers to finish.
@@ -374,7 +460,7 @@ public:
                 List<Container>& recvData,
                 const int tag = UPstream::msgType(),
                 const label comm = UPstream::worldComm,
-                const bool block = true
+                const bool wait = true  //!< Wait for requests to complete
             );
 };
 
@@ -386,10 +472,11 @@ public:
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
 #ifdef NoRepository
-    #include "gatherScatter.C"
+    #include "PstreamBroadcast.C"
-    #include "combineGatherScatter.C"
+    #include "PstreamGather.C"
-    #include "gatherScatterList.C"
+    #include "PstreamCombineGather.C"
-    #include "exchange.C"
+    #include "PstreamGatherList.C"
+    #include "PstreamExchange.C"
 #endif
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //

src/OpenFOAM/db/IOstreams/Pstreams/PstreamBroadcast.C

@@ -0,0 +1,142 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | www.openfoam.com
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+    Copyright (C) 2022 OpenCFD Ltd.
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "OPstream.H"
+#include "IPstream.H"
+#include "contiguous.H"
+
+// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * //
+
+template<class T>
+void Foam::Pstream::genericBroadcast(T& value, const label comm)
+{
+    // Generic: use stream interface
+    if (UPstream::parRun() && UPstream::nProcs(comm) > 1)
+    {
+        if (UPstream::master(comm))
+        {
+            OPBstream toAll(UPstream::masterNo(), comm);
+            toAll << value;
+        }
+        else
+        {
+            IPBstream fromMaster(UPstream::masterNo(), comm);
+            fromMaster >> value;
+        }
+    }
+}
+
+
+template<class T>
+void Foam::Pstream::broadcast(T& value, const label comm)
+{
+    if (!is_contiguous<T>::value)
+    {
+        Pstream::genericBroadcast(value, comm);
+    }
+    else if (UPstream::parRun() && UPstream::nProcs(comm) > 1)
+    {
+        UPstream::broadcast
+        (
+            reinterpret_cast<char*>(&value),
+            sizeof(T),
+            comm,
+            UPstream::masterNo()
+        );
+    }
+}
+
+
+template<class T>
+void Foam::Pstream::broadcast(List<T>& values, const label comm)
+{
+    if (!is_contiguous<T>::value)
+    {
+        Pstream::genericBroadcast(values, comm);
+    }
+    else if (UPstream::parRun() && UPstream::nProcs(comm) > 1)
+    {
+        // Broadcast the size
+        label len(values.size());
+        UPstream::broadcast
+        (
+            reinterpret_cast<char*>(&len),
+            sizeof(label),
+            comm,
+            UPstream::masterNo()
+        );
+        values.resize_nocopy(len);  // A no-op on master
+
+        if (len)
+        {
+            UPstream::broadcast
+            (
+                values.data_bytes(),
+                values.size_bytes(),
+                comm,
+                UPstream::masterNo()
+            );
+        }
+    }
+}
+
+
+template<class T, int SizeMin>
+void Foam::Pstream::broadcast(DynamicList<T, SizeMin>& values, const label comm)
+{
+    if (!is_contiguous<T>::value)
+    {
+        Pstream::genericBroadcast(values, comm);
+    }
+    else if (UPstream::parRun() && UPstream::nProcs(comm) > 1)
+    {
+        // Broadcast the size
+        label len(values.size());
+        UPstream::broadcast
+        (
+            reinterpret_cast<char*>(&len),
+            sizeof(label),
+            comm,
+            UPstream::masterNo()
+        );
+        values.resize_nocopy(len);  // A no-op on master
+
+        if (len)
+        {
+            UPstream::broadcast
+            (
+                values.data_bytes(),
+                values.size_bytes(),
+                comm,
+                UPstream::masterNo()
+            );
+        }
+    }
+}
+
+
+// ************************************************************************* //

src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2017 OpenFOAM Foundation
-    Copyright (C) 2021 OpenCFD Ltd.
+    Copyright (C) 2021-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -27,6 +27,74 @@ License
 \*---------------------------------------------------------------------------*/
 
 #include "PstreamBuffers.H"
+#include "bitSet.H"
+
+// * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
+
+void Foam::PstreamBuffers::finalExchange
+(
+    labelList& recvSizes,
+    const bool wait
+)
+{
+    // Could also check that it is not called twice
+    // but that is used for overlapping send/recv (eg, overset)
+    finishedSendsCalled_ = true;
+
+    if (commsType_ == UPstream::commsTypes::nonBlocking)
+    {
+        // all-to-all
+        Pstream::exchangeSizes(sendBuf_, recvSizes, comm_);
+
+        Pstream::exchange<DynamicList<char>, char>
+        (
+            sendBuf_,
+            recvSizes,
+            recvBuf_,
+            tag_,
+            comm_,
+            wait
+        );
+    }
+}
+
+
+void Foam::PstreamBuffers::finalExchange
+(
+    const labelUList& sendProcs,
+    const labelUList& recvProcs,
+    labelList& recvSizes,
+    const bool wait
+)
+{
+    // Could also check that it is not called twice
+    // but that is used for overlapping send/recv (eg, overset)
+    finishedSendsCalled_ = true;
+
+    if (commsType_ == UPstream::commsTypes::nonBlocking)
+    {
+        Pstream::exchangeSizes
+        (
+            sendProcs,
+            recvProcs,
+            sendBuf_,
+            recvSizes,
+            tag_,
+            comm_
+        );
+
+        Pstream::exchange<DynamicList<char>, char>
+        (
+            sendBuf_,
+            recvSizes,
+            recvBuf_,
+            tag_,
+            comm_,
+            wait
+        );
+    }
+}
+
 
 // * * * * * * * * * * * * * * * * Constructor * * * * * * * * * * * * * * * //
 
@@ -38,14 +106,15 @@ Foam::PstreamBuffers::PstreamBuffers
     IOstreamOption::streamFormat fmt
 )
 :
+    finishedSendsCalled_(false),
+    allowClearRecv_(true),
+    format_(fmt),
     commsType_(commsType),
     tag_(tag),
     comm_(comm),
-    format_(fmt),
     sendBuf_(UPstream::nProcs(comm)),
     recvBuf_(UPstream::nProcs(comm)),
-    recvBufPos_(UPstream::nProcs(comm), Zero),
+    recvBufPos_(UPstream::nProcs(comm), Zero)
-    finishedSendsCalled_(false)
 {}
 
 
@@ -70,45 +139,125 @@ Foam::PstreamBuffers::~PstreamBuffers()
 
 // * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * * //
 
-void Foam::PstreamBuffers::finishedSends(const bool block)
+void Foam::PstreamBuffers::clear()
 {
-    // Could also check that it is not called twice
+    for (DynamicList<char>& buf : sendBuf_)
-    finishedSendsCalled_ = true;
-
-    if (commsType_ == UPstream::commsTypes::nonBlocking)
     {
-        Pstream::exchange<DynamicList<char>, char>
+        buf.clear();
-        (
-            sendBuf_,
-            recvBuf_,
-            tag_,
-            comm_,
-            block
-        );
     }
+    for (DynamicList<char>& buf : recvBuf_)
+    {
+        buf.clear();
+    }
+    recvBufPos_ = 0;
+
+    finishedSendsCalled_ = false;
 }
 
 
-void Foam::PstreamBuffers::finishedSends(labelList& recvSizes, const bool block)
+void Foam::PstreamBuffers::clearStorage()
 {
-    // Could also check that it is not called twice
+    // Could also clear out entire sendBuf_, recvBuf_ and reallocate.
-    finishedSendsCalled_ = true;
+    // Not sure if it makes much difference
-
+    for (DynamicList<char>& buf : sendBuf_)
-    if (commsType_ == UPstream::commsTypes::nonBlocking)
     {
-        Pstream::exchangeSizes(sendBuf_, recvSizes, comm_);
+        buf.clearStorage();
-
-        Pstream::exchange<DynamicList<char>, char>
-        (
-            sendBuf_,
-            recvSizes,
-            recvBuf_,
-            tag_,
-            comm_,
-            block
-        );
     }
+    for (DynamicList<char>& buf : recvBuf_)
+    {
+        buf.clearStorage();
+    }
+    recvBufPos_ = 0;
+
+    finishedSendsCalled_ = false;
+}
+
+
+bool Foam::PstreamBuffers::hasSendData() const
+{
+    for (const DynamicList<char>& buf : sendBuf_)
+    {
+        if (!buf.empty())
+        {
+            return true;
+        }
+    }
+    return false;
+}
+
+
+bool Foam::PstreamBuffers::hasSendData(const label proci) const
+{
+    return !sendBuf_[proci].empty();
+}
+
+
+bool Foam::PstreamBuffers::hasRecvData() const
+{
+    if (finishedSendsCalled_)
+    {
+        for (const DynamicList<char>& buf : recvBuf_)
+        {
+            if (!buf.empty())
+            {
+                return true;
+            }
+        }
+    }
+    #ifdef FULLDEBUG
     else
+    {
+        FatalErrorInFunction
+            << "Call finishedSends first" << exit(FatalError);
+    }
+    #endif
+
+    return false;
+}
+
+
+bool Foam::PstreamBuffers::hasRecvData(const label proci) const
+{
+    if (finishedSendsCalled_)
+    {
+        return !recvBuf_[proci].empty();
+    }
+    #ifdef FULLDEBUG
+    else
+    {
+        FatalErrorInFunction
+            << "Call finishedSends first" << exit(FatalError);
+    }
+    #endif
+
+    return false;
+}
+
+
+bool Foam::PstreamBuffers::allowClearRecv(bool on) noexcept
+{
+    bool old(allowClearRecv_);
+    allowClearRecv_ = on;
+    return old;
+}
+
+
+void Foam::PstreamBuffers::finishedSends(const bool wait)
+{
+    labelList recvSizes;
+    finalExchange(recvSizes, wait);
+}
+
+
+void Foam::PstreamBuffers::finishedSends
+(
+    labelList& recvSizes,
+    const bool wait
+)
+{
+    finalExchange(recvSizes, wait);
+
+    if (commsType_ != UPstream::commsTypes::nonBlocking)
     {
         FatalErrorInFunction
             << "Obtaining sizes not supported in "
@@ -122,18 +271,107 @@ void Foam::PstreamBuffers::finishedSends(labelList& recvSizes, const bool block)
 }
 
 
-void Foam::PstreamBuffers::clear()
+void Foam::PstreamBuffers::finishedSends
+(
+    const labelUList& sendProcs,
+    const labelUList& recvProcs,
+    const bool wait
+)
 {
-    for (DynamicList<char>& buf : sendBuf_)
+    labelList recvSizes;
-    {
+    finalExchange(sendProcs, recvProcs, recvSizes, wait);
-        buf.clear();
 }
-    for (DynamicList<char>& buf : recvBuf_)
+
+
+void Foam::PstreamBuffers::finishedSends
+(
+    const labelUList& sendProcs,
+    const labelUList& recvProcs,
+    labelList& recvSizes,
+    const bool wait
+)
 {
-        buf.clear();
+    finalExchange(sendProcs, recvProcs, recvSizes, wait);
+
+    if (commsType_ != UPstream::commsTypes::nonBlocking)
+    {
+        FatalErrorInFunction
+            << "Obtaining sizes not supported in "
+            << UPstream::commsTypeNames[commsType_] << endl
+            << " since transfers already in progress. Use non-blocking instead."
+            << exit(FatalError);
+
+        // Note: maybe possible only if using different tag from write started
+        // by ~UOPstream. Needs some work.
     }
-    recvBufPos_ = 0;
+}
-    finishedSendsCalled_ = false;
+
+
+bool Foam::PstreamBuffers::finishedSends
+(
+    bitSet& sendConnections,
+    DynamicList<label>& sendProcs,
+    DynamicList<label>& recvProcs,
+    const bool wait
+)
+{
+    bool changed = (sendConnections.size() != nProcs());
+
+    if (changed)
+    {
+        sendConnections.resize(nProcs());
+    }
+
+    // Update send connections
+    // - reasonable to assume there are no self-sends on UPstream::myProcNo
+    forAll(sendBuf_, proci)
+    {
+        // ie, hasSendData(proci)
+        if (sendConnections.set(proci, !sendBuf_[proci].empty()))
+        {
+            // The state changed
+            changed = true;
+        }
+    }
+
+    reduce(changed, orOp<bool>());
+
+    if (changed)
+    {
+        // Create send/recv topology
+
+        // The send ranks
+        sendProcs.clear();
+        forAll(sendBuf_, proci)
+        {
+            // ie, hasSendData(proci)
+            if (!sendBuf_[proci].empty())
+            {
+                sendProcs.append(proci);
+            }
+        }
+
+        finishedSends(wait);  // All-to-all
+
+        // The recv ranks
+        recvProcs.clear();
+        forAll(recvBuf_, proci)
+        {
+            // ie, hasRecvData(proci)
+            if (!recvBuf_[proci].empty())
+            {
+                recvProcs.append(proci);
+            }
+        }
+    }
+    else
+    {
+        // Use existing send/recv ranks
+
+        finishedSends(sendProcs, recvProcs, wait);
+    }
+
+    return changed;
 }
 
 

src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.H

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2017 OpenFOAM Foundation
-    Copyright (C) 2021 OpenCFD Ltd.
+    Copyright (C) 2021-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -70,8 +70,8 @@ SourceFiles
 
 #include "Pstream.H"
 
-#ifndef PstreamBuffers_H
+#ifndef Foam_PstreamBuffers_H
-#define PstreamBuffers_H
+#define Foam_PstreamBuffers_H
 
 #include "DynamicList.H"
 #include "UPstream.H"
@@ -82,17 +82,31 @@ SourceFiles
 namespace Foam
 {
 
+// Forward Declarations
+class bitSet;
+
 /*---------------------------------------------------------------------------*\
                        Class PstreamBuffers Declaration
 \*---------------------------------------------------------------------------*/
 
 class PstreamBuffers
 {
-    friend class UOPstream;
+    // Friendship
-    friend class UIPstream;
+    friend class UOPstreamBase;  // Access to sendBuf_
+    friend class UIPstreamBase;  // Access to recvBuf_, recvBufPos_
+
 
     // Private Data
 
+        //- Track if sends are complete
+        bool finishedSendsCalled_;
+
+        //- Permit clear of individual receive buffer by external access
+        bool allowClearRecv_;
+
+        //- Buffer format (ascii | binary)
+        const IOstreamOption::streamFormat format_;
+
         //- Communications type of this stream
         const UPstream::commsTypes commsType_;
 
@@ -102,20 +116,32 @@ class PstreamBuffers
         //- Communicator
         const label comm_;
 
-        //- Buffer format (ascii | binary)
+        //- Send buffer. Size is nProcs()
-        const IOstreamOption::streamFormat format_;
-
-        //- Send buffer
         List<DynamicList<char>> sendBuf_;
 
-        //- Receive buffer
+        //- Receive buffer. Size is nProcs()
         List<DynamicList<char>> recvBuf_;
 
-        //- Current read positions within recvBuf_
+        //- Current read positions within recvBuf_. Size is nProcs()
         labelList recvBufPos_;
 
-        //- Track if sends are complete
+
-        bool finishedSendsCalled_;
+    // Private Member Functions
+
+        //- Mark all sends as having been done.
+        //  This will start receives (nonBlocking comms).
+        void finalExchange(labelList& recvSizes, const bool wait);
+
+        //- Mark sends as done.
+        //  Only exchange sizes using the sendProcs/recvProcs subset
+        //  (nonBlocking comms).
+        void finalExchange
+        (
+            const labelUList& sendProcs,
+            const labelUList& recvProcs,
+            labelList& recvSizes,
+            const bool wait
+        );
 
 
 public:
@@ -138,6 +164,20 @@ public:
 
     // Member Functions
 
+    // Access
+
+        //- The associated buffer format (ascii | binary)
+        IOstreamOption::streamFormat format() const noexcept
+        {
+            return format_;
+        }
+
+        //- The communications type of the stream
+        UPstream::commsTypes commsType() const noexcept
+        {
+            return commsType_;
+        }
+
         //- The transfer message type
         int tag() const noexcept
         {
@@ -150,20 +190,176 @@ public:
             return comm_;
         }
 
-        //- Mark all sends as having been done.
-        //  This will start receives in non-blocking mode.
-        //  If block will wait for all transfers to finish
-        //  (only relevant for nonBlocking mode)
-        void finishedSends(const bool block = true);
 
-        //- Mark all sends as having been done.
+    // Sizing
-        //  Same as above but also returns sizes (bytes) received.
-        //  \note currently only valid for non-blocking.
-        void finishedSends(labelList& recvSizes, const bool block = true);
 
-        //- Reset (clear) individual buffers and reset state.
+        //- Number of ranks associated with PstreamBuffers
-        //  Does not clear buffer storage
+        label nProcs() const noexcept
+        {
+            return recvBufPos_.size();
+        }
+
+        //- Range of ranks indices associated with PstreamBuffers
+        UPstream::rangeType allProcs() const noexcept
+        {
+            // Proc 0 -> nProcs (int value)
+            return UPstream::rangeType(static_cast<int>(nProcs()));
+        }
+
+
+    // Queries
+
+        //- True if finishedSends() or finishedNeighbourSends() has been called
+        bool finished() const noexcept
+        {
+            return finishedSendsCalled_;
+        }
+
+        //- True if any (local) send buffers have data
+        bool hasSendData() const;
+
+        //- True if (local) send buffer has data on specified processor.
+        bool hasSendData(const label proci) const;
+
+        //- True if any (local) recv buffers have data.
+        //- Must call finishedSends() or finishedNeighbourSends() first!
+        bool hasRecvData() const;
+
+        //- True if (local) recv buffer has data on specified processor.
+        //- Must call finishedSends() or finishedNeighbourSends() first!
+        bool hasRecvData(const label proci) const;
+
+        //- Is clearStorage of individual receive buffer by external hooks
+        //- allowed? (default: true)
+        bool allowClearRecv() const noexcept
+        {
+            return allowClearRecv_;
+        }
+
+
+    // Edit
+
+        //- Clear individual buffers and reset states.
+        //  Does not clear individual buffer storage.
         void clear();
+
+        //- Clear individual buffer storage and reset states.
+        void clearStorage();
+
+        //- Change allowClearRecv, return previous value
+        bool allowClearRecv(bool on) noexcept;
+
+
+    // Functions
+
+        //- Mark sends as done
+        //
+        //  Non-blocking mode: populates receive buffers (all-to-all).
+        //  \param wait wait for requests to complete (in nonBlocking mode)
+        void finishedSends(const bool wait = true);
+
+        //- Mark sends as done.
+        //- Recovers the sizes (bytes) received.
+        //
+        //  Non-blocking mode: populates receive buffers (all-to-all).
+        //  \param[out] recvSizes the sizes (bytes) received
+        //  \param wait wait for requests to complete (in nonBlocking mode)
+        //
+        //  \warning currently only valid for nonBlocking comms.
+        void finishedSends(labelList& recvSizes, const bool wait = true);
+
+        //- Mark sends as done using subset of send/recv ranks
+        //- to exchange data on.
+        //
+        //  Non-blocking mode: populates receive buffers.
+        //  \param sendProcs ranks used for sends
+        //  \param recvProcs ranks used for recvs
+        //  \param wait wait for requests to complete (in nonBlocking mode)
+        //
+        //  \warning currently only valid for nonBlocking comms.
+        void finishedSends
+        (
+            const labelUList& sendProcs,
+            const labelUList& recvProcs,
+            const bool wait = true
+        );
+
+        //- Mark sends as done using subset of send/recv ranks
+        //- to exchange data on. Recovers the sizes (bytes) received.
+        //
+        //  Non-blocking mode: populates receive buffers.
+        //
+        //  \param sendProcs ranks used for sends
+        //  \param recvProcs ranks used for recvs
+        //  \param[out] recvSizes the sizes (bytes) received
+        //  \param wait wait for requests to complete (in nonBlocking mode)
+        //
+        //  \warning currently only valid for nonBlocking comms.
+        void finishedSends
+        (
+            const labelUList& sendProcs,
+            const labelUList& recvProcs,
+            labelList& recvSizes,
+            const bool wait = true
+        );
+
+        //- A caching version that uses a limited send/recv connectivity.
+        //
+        //  Non-blocking mode: populates receive buffers.
+        //  \param sendConnections on/off for sending ranks
+        //  \param sendProcs ranks used for sends
+        //  \param recvProcs ranks used for recvs
+        //  \param wait wait for requests to complete (in nonBlocking mode)
+        //
+        //  \return True if the send/recv connectivity changed
+        //
+        //  \warning currently only valid for nonBlocking comms.
+        bool finishedSends
+        (
+            bitSet& sendConnections,
+            DynamicList<label>& sendProcs,
+            DynamicList<label>& recvProcs,
+            const bool wait = true
+        );
+
+        //- Mark sends as done using subset of send/recv ranks
+        //- and recover the sizes (bytes) received.
+        //
+        //  Non-blocking mode: populates receive buffers.
+        //
+        //  \param neighProcs ranks used for sends/recvs
+        //  \param wait wait for requests to complete (in nonBlocking mode)
+        //
+        //  \warning currently only valid for nonBlocking comms.
+        //  \note Same as finishedSends with identical sendProcs/recvProcs
+        void finishedNeighbourSends
+        (
+            const labelUList& neighProcs,
+            const bool wait = true
+        )
+        {
+            finishedSends(neighProcs, neighProcs, wait);
+        }
+
+        //- Mark sends as done using subset of send/recv ranks
+        //- and recover the sizes (bytes) received.
+        //
+        //  Non-blocking mode: it will populate receive buffers.
+        //
+        //  \param neighProcs ranks used for sends/recvs
+        //  \param[out] recvSizes the sizes (bytes) received
+        //  \param wait wait for requests to complete (in nonBlocking mode)
+        //
+        //  \warning currently only valid for nonBlocking mode.
+        void finishedNeighbourSends
+        (
+            const labelUList& neighProcs,
+            labelList& recvSizes,
+            const bool wait = true
+        )
+        {
+            finishedSends(neighProcs, neighProcs, recvSizes, wait);
+        }
 };
 
 

src/OpenFOAM/db/IOstreams/Pstreams/PstreamCombineGather.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2017 OpenFOAM Foundation
-    Copyright (C) 2019-2021 OpenCFD Ltd.
+    Copyright (C) 2019-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -27,7 +27,7 @@ License
 Description
     Variant of gather, scatter.
     Normal gather uses:
-    - construct null and read (>>) from Istream
+    - default construct and read (>>) from Istream
     - binary operator and assignment operator to combine values
 
     combineGather uses:
@@ -47,7 +47,7 @@ template<class T, class CombineOp>
 void Foam::Pstream::combineGather
 (
     const List<UPstream::commsStruct>& comms,
-    T& Value,
+    T& value,
     const CombineOp& cop,
     const int tag,
     const label comm
@@ -55,22 +55,21 @@ void Foam::Pstream::combineGather
 {
     if (UPstream::parRun() && UPstream::nProcs(comm) > 1)
     {
-        // Get my communication order
+        // My communication order
         const commsStruct& myComm = comms[UPstream::myProcNo(comm)];
 
         // Receive from my downstairs neighbours
-        forAll(myComm.below(), belowI)
+        for (const label belowID : myComm.below())
         {
-            label belowID = myComm.below()[belowI];
-
             if (is_contiguous<T>::value)
             {
-                T value;
+                T received;
+
                 UIPstream::read
                 (
                     UPstream::commsTypes::scheduled,
                     belowID,
-                    reinterpret_cast<char*>(&value),
+                    reinterpret_cast<char*>(&received),
                     sizeof(T),
                     tag,
                     comm
@@ -79,10 +78,10 @@ void Foam::Pstream::combineGather
                 if (debug & 2)
                 {
                     Pout<< " received from "
-                        << belowID << " data:" << value << endl;
+                        << belowID << " data:" << received << endl;
                 }
 
-                cop(Value, value);
+                cop(value, received);
             }
             else
             {
@@ -94,25 +93,25 @@ void Foam::Pstream::combineGather
                     tag,
                     comm
                 );
-                T value(fromBelow);
+                T received(fromBelow);
 
                 if (debug & 2)
                 {
                     Pout<< " received from "
-                        << belowID << " data:" << value << endl;
+                        << belowID << " data:" << received << endl;
                 }
 
-                cop(Value, value);
+                cop(value, received);
             }
         }
 
-        // Send up Value
+        // Send up value
         if (myComm.above() != -1)
         {
             if (debug & 2)
             {
                 Pout<< " sending to " << myComm.above()
-                    << " data:" << Value << endl;
+                    << " data:" << value << endl;
             }
 
             if (is_contiguous<T>::value)
@@ -121,7 +120,7 @@ void Foam::Pstream::combineGather
                 (
                     UPstream::commsTypes::scheduled,
                     myComm.above(),
-                    reinterpret_cast<const char*>(&Value),
+                    reinterpret_cast<const char*>(&value),
                     sizeof(T),
                     tag,
                     comm
@@ -137,7 +136,7 @@ void Foam::Pstream::combineGather
                     tag,
                     comm
                 );
-                toAbove << Value;
+                toAbove << value;
             }
         }
     }
@@ -147,49 +146,38 @@ void Foam::Pstream::combineGather
 template<class T, class CombineOp>
 void Foam::Pstream::combineGather
 (
-    T& Value,
+    T& value,
     const CombineOp& cop,
     const int tag,
     const label comm
 )
-{
-    if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
 {
     combineGather
     (
-            UPstream::linearCommunication(comm),
+        UPstream::whichCommunication(comm),
-            Value,
+        value,
         cop,
         tag,
         comm
     );
 }
-    else
-    {
-        combineGather
-        (
-            UPstream::treeCommunication(comm),
-            Value,
-            cop,
-            tag,
-            comm
-        );
-    }
-}
 
 
 template<class T>
 void Foam::Pstream::combineScatter
 (
     const List<UPstream::commsStruct>& comms,
-    T& Value,
+    T& value,
     const int tag,
     const label comm
 )
 {
+    #ifndef Foam_Pstream_scatter_nobroadcast
+    Pstream::broadcast(value, comm);
+    #else
     if (UPstream::parRun() && UPstream::nProcs(comm) > 1)
     {
-        // Get my communication order
+        // My communication order
         const UPstream::commsStruct& myComm = comms[UPstream::myProcNo(comm)];
 
         // Receive from up
@@ -201,7 +189,7 @@ void Foam::Pstream::combineScatter
                 (
                     UPstream::commsTypes::scheduled,
                     myComm.above(),
-                    reinterpret_cast<char*>(&Value),
+                    reinterpret_cast<char*>(&value),
                     sizeof(T),
                     tag,
                     comm
@@ -217,25 +205,14 @@ void Foam::Pstream::combineScatter
                     tag,
                     comm
                 );
-                Value = T(fromAbove);
+                value = T(fromAbove);
-            }
-
-            if (debug & 2)
-            {
-                Pout<< " received from "
-                    << myComm.above() << " data:" << Value << endl;
             }
         }
 
         // Send to my downstairs neighbours
         forAllReverse(myComm.below(), belowI)
         {
-            label belowID = myComm.below()[belowI];
+            const label belowID = myComm.below()[belowI];
-
-            if (debug & 2)
-            {
-                Pout<< " sending to " << belowID << " data:" << Value << endl;
-            }
 
             if (is_contiguous<T>::value)
             {
@@ -243,7 +220,7 @@ void Foam::Pstream::combineScatter
                 (
                     UPstream::commsTypes::scheduled,
                     belowID,
-                    reinterpret_cast<const char*>(&Value),
+                    reinterpret_cast<const char*>(&value),
                     sizeof(T),
                     tag,
                     comm
@@ -259,29 +236,27 @@ void Foam::Pstream::combineScatter
                     tag,
                     comm
                 );
-                toBelow << Value;
+                toBelow << value;
             }
         }
     }
+    #endif
 }
 
 
 template<class T>
 void Foam::Pstream::combineScatter
 (
-    T& Value,
+    T& value,
     const int tag,
     const label comm
 )
 {
-    if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
+    #ifndef Foam_Pstream_scatter_nobroadcast
-    {
+    Pstream::broadcast(value, comm);
-        combineScatter(UPstream::linearCommunication(comm), Value, tag, comm);
+    #else
-    }
+    combineScatter(UPstream::whichCommunication(comm), value, tag, comm);
-    else
+    #endif
-    {
-        combineScatter(UPstream::treeCommunication(comm), Value, tag, comm);
-    }
 }
 
 
@@ -289,7 +264,7 @@ template<class T, class CombineOp>
 void Foam::Pstream::listCombineGather
 (
     const List<UPstream::commsStruct>& comms,
-    List<T>& Values,
+    List<T>& values,
     const CombineOp& cop,
     const int tag,
     const label comm
@@ -297,24 +272,22 @@ void Foam::Pstream::listCombineGather
 {
     if (UPstream::parRun() && UPstream::nProcs(comm) > 1)
     {
-        // Get my communication order
+        // My communication order
         const commsStruct& myComm = comms[UPstream::myProcNo(comm)];
 
         // Receive from my downstairs neighbours
-        forAll(myComm.below(), belowI)
+        for (const label belowID : myComm.below())
         {
-            label belowID = myComm.below()[belowI];
-
             if (is_contiguous<T>::value)
             {
-                List<T> receivedValues(Values.size());
+                List<T> received(values.size());
 
                 UIPstream::read
                 (
                     UPstream::commsTypes::scheduled,
                     belowID,
-                    receivedValues.data_bytes(),
+                    received.data_bytes(),
-                    receivedValues.size_bytes(),
+                    received.size_bytes(),
                     tag,
                     comm
                 );
@@ -322,12 +295,12 @@ void Foam::Pstream::listCombineGather
                 if (debug & 2)
                 {
                     Pout<< " received from "
-                        << belowID << " data:" << receivedValues << endl;
+                        << belowID << " data:" << received << endl;
                 }
 
-                forAll(Values, i)
+                forAll(values, i)
                 {
-                    cop(Values[i], receivedValues[i]);
+                    cop(values[i], received[i]);
                 }
             }
             else
@@ -340,28 +313,28 @@ void Foam::Pstream::listCombineGather
                     tag,
                     comm
                 );
-                List<T> receivedValues(fromBelow);
+                List<T> received(fromBelow);
 
                 if (debug & 2)
                 {
                     Pout<< " received from "
-                        << belowID << " data:" << receivedValues << endl;
+                        << belowID << " data:" << received << endl;
                 }
 
-                forAll(Values, i)
+                forAll(values, i)
                 {
-                    cop(Values[i], receivedValues[i]);
+                    cop(values[i], received[i]);
                 }
             }
         }
 
-        // Send up Value
+        // Send up values
         if (myComm.above() != -1)
         {
             if (debug & 2)
             {
                 Pout<< " sending to " << myComm.above()
-                    << " data:" << Values << endl;
+                    << " data:" << values << endl;
             }
 
             if (is_contiguous<T>::value)
@@ -370,8 +343,8 @@ void Foam::Pstream::listCombineGather
                 (
                     UPstream::commsTypes::scheduled,
                     myComm.above(),
-                    Values.cdata_bytes(),
+                    values.cdata_bytes(),
-                    Values.size_bytes(),
+                    values.size_bytes(),
                     tag,
                     comm
                 );
@@ -386,7 +359,7 @@ void Foam::Pstream::listCombineGather
                     tag,
                     comm
                 );
-                toAbove << Values;
+                toAbove << values;
             }
         }
     }
@@ -396,49 +369,38 @@ void Foam::Pstream::listCombineGather
 template<class T, class CombineOp>
 void Foam::Pstream::listCombineGather
 (
-    List<T>& Values,
+    List<T>& values,
     const CombineOp& cop,
     const int tag,
     const label comm
 )
-{
-    if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
 {
     listCombineGather
     (
-            UPstream::linearCommunication(comm),
+        UPstream::whichCommunication(comm),
-            Values,
+        values,
         cop,
         tag,
         comm
     );
 }
-    else
-    {
-        listCombineGather
-        (
-            UPstream::treeCommunication(comm),
-            Values,
-            cop,
-            tag,
-            comm
-        );
-    }
-}
 
 
 template<class T>
 void Foam::Pstream::listCombineScatter
 (
     const List<UPstream::commsStruct>& comms,
-    List<T>& Values,
+    List<T>& values,
     const int tag,
     const label comm
 )
 {
+    #ifndef Foam_Pstream_scatter_nobroadcast
+    Pstream::broadcast(values, comm);
+    #else
     if (UPstream::parRun() && UPstream::nProcs(comm) > 1)
     {
-        // Get my communication order
+        // My communication order
         const UPstream::commsStruct& myComm = comms[UPstream::myProcNo(comm)];
 
         // Receive from up
@@ -450,8 +412,8 @@ void Foam::Pstream::listCombineScatter
                 (
                     UPstream::commsTypes::scheduled,
                     myComm.above(),
-                    Values.data_bytes(),
+                    values.data_bytes(),
-                    Values.size_bytes(),
+                    values.size_bytes(),
                     tag,
                     comm
                 );
@@ -466,25 +428,14 @@ void Foam::Pstream::listCombineScatter
                     tag,
                     comm
                 );
-                fromAbove >> Values;
+                fromAbove >> values;
-            }
-
-            if (debug & 2)
-            {
-                Pout<< " received from "
-                    << myComm.above() << " data:" << Values << endl;
             }
         }
 
         // Send to my downstairs neighbours
         forAllReverse(myComm.below(), belowI)
         {
-            label belowID = myComm.below()[belowI];
+            const label belowID = myComm.below()[belowI];
-
-            if (debug & 2)
-            {
-                Pout<< " sending to " << belowID << " data:" << Values << endl;
-            }
 
             if (is_contiguous<T>::value)
             {
@@ -492,8 +443,8 @@ void Foam::Pstream::listCombineScatter
                 (
                     UPstream::commsTypes::scheduled,
                     belowID,
-                    Values.cdata_bytes(),
+                    values.cdata_bytes(),
-                    Values.size_bytes(),
+                    values.size_bytes(),
                     tag,
                     comm
                 );
@@ -508,41 +459,33 @@ void Foam::Pstream::listCombineScatter
                     tag,
                     comm
                 );
-                toBelow << Values;
+                toBelow << values;
             }
         }
     }
+    #endif
 }
 
 
 template<class T>
 void Foam::Pstream::listCombineScatter
 (
-    List<T>& Values,
+    List<T>& values,
     const int tag,
     const label comm
 )
 {
-    if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
+    #ifndef Foam_Pstream_scatter_nobroadcast
-    {
+    Pstream::broadcast(values, comm);
+    #else
     listCombineScatter
     (
-            UPstream::linearCommunication(comm),
+        UPstream::whichCommunication(comm),
-            Values,
+        values,
         tag,
         comm
     );
-    }
+    #endif
-    else
-    {
-        listCombineScatter
-        (
-            UPstream::treeCommunication(comm),
-            Values,
-            tag,
-            comm
-        );
-    }
 }
 
 
@@ -550,7 +493,7 @@ template<class Container, class CombineOp>
 void Foam::Pstream::mapCombineGather
 (
     const List<UPstream::commsStruct>& comms,
-    Container& Values,
+    Container& values,
     const CombineOp& cop,
     const int tag,
     const label comm
@@ -558,13 +501,13 @@ void Foam::Pstream::mapCombineGather
 {
     if (UPstream::parRun() && UPstream::nProcs(comm) > 1)
     {
-        // Get my communication order
+        // My communication order
         const commsStruct& myComm = comms[UPstream::myProcNo(comm)];
 
         // Receive from my downstairs neighbours
-        forAll(myComm.below(), belowI)
+        for (const label belowID : myComm.below())
         {
-            label belowID = myComm.below()[belowI];
+            // Map/HashTable: non-contiguous
 
             IPstream fromBelow
             (
@@ -574,43 +517,43 @@ void Foam::Pstream::mapCombineGather
                 tag,
                 comm
             );
-            Container receivedValues(fromBelow);
+            Container received(fromBelow);
 
             if (debug & 2)
             {
                 Pout<< " received from "
-                    << belowID << " data:" << receivedValues << endl;
+                    << belowID << " data:" << received << endl;
             }
 
             for
             (
-                typename Container::const_iterator slaveIter =
+                auto recvIter = received.cbegin();
-                    receivedValues.begin();
+                recvIter != received.cend();
-                slaveIter != receivedValues.end();
+                ++recvIter
-                ++slaveIter
             )
             {
-                typename Container::iterator
+                auto masterIter = values.find(recvIter.key());
-                    masterIter = Values.find(slaveIter.key());
 
-                if (masterIter != Values.end())
+                if (masterIter != values.end())  // == found()
                 {
-                    cop(masterIter(), slaveIter());
+                    // Combine with existing
+                    cop(masterIter.val(), recvIter.val());
                 }
                 else
                 {
-                    Values.insert(slaveIter.key(), slaveIter());
+                    // Insert new key/value
+                    values.insert(recvIter.key(), recvIter.val());
                 }
             }
         }
 
-        // Send up Value
+        // Send up values
         if (myComm.above() != -1)
         {
             if (debug & 2)
             {
                 Pout<< " sending to " << myComm.above()
-                    << " data:" << Values << endl;
+                    << " data:" << values << endl;
             }
 
             OPstream toAbove
@@ -621,7 +564,7 @@ void Foam::Pstream::mapCombineGather
                 tag,
                 comm
             );
-            toAbove << Values;
+            toAbove << values;
         }
     }
 }
@@ -630,49 +573,38 @@ void Foam::Pstream::mapCombineGather
 template<class Container, class CombineOp>
 void Foam::Pstream::mapCombineGather
 (
-    Container& Values,
+    Container& values,
     const CombineOp& cop,
     const int tag,
     const label comm
 )
-{
-    if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
 {
     mapCombineGather
     (
-            UPstream::linearCommunication(comm),
+        UPstream::whichCommunication(comm),
-            Values,
+        values,
         cop,
         tag,
         comm
     );
 }
-    else
-    {
-        mapCombineGather
-        (
-            UPstream::treeCommunication(comm),
-            Values,
-            cop,
-            tag,
-            comm
-        );
-    }
-}
 
 
 template<class Container>
 void Foam::Pstream::mapCombineScatter
 (
     const List<UPstream::commsStruct>& comms,
-    Container& Values,
+    Container& values,
     const int tag,
     const label comm
 )
 {
+    #ifndef Foam_Pstream_scatter_nobroadcast
+    Pstream::broadcast(values, comm);
+    #else
     if (UPstream::parRun() && UPstream::nProcs(comm) > 1)
     {
-        // Get my communication order
+        // My communication order
         const UPstream::commsStruct& myComm = comms[UPstream::myProcNo(comm)];
 
         // Receive from up
@@ -686,23 +618,23 @@ void Foam::Pstream::mapCombineScatter
                 tag,
                 comm
             );
-            fromAbove >> Values;
+            fromAbove >> values;
 
             if (debug & 2)
             {
                 Pout<< " received from "
-                    << myComm.above() << " data:" << Values << endl;
+                    << myComm.above() << " data:" << values << endl;
             }
         }
 
         // Send to my downstairs neighbours
         forAllReverse(myComm.below(), belowI)
         {
-            label belowID = myComm.below()[belowI];
+            const label belowID = myComm.below()[belowI];
 
             if (debug & 2)
             {
-                Pout<< " sending to " << belowID << " data:" << Values << endl;
+                Pout<< " sending to " << belowID << " data:" << values << endl;
             }
 
             OPstream toBelow
@@ -713,40 +645,32 @@ void Foam::Pstream::mapCombineScatter
                 tag,
                 comm
             );
-            toBelow << Values;
+            toBelow << values;
         }
     }
+    #endif
 }
 
 
 template<class Container>
 void Foam::Pstream::mapCombineScatter
 (
-    Container& Values,
+    Container& values,
     const int tag,
     const label comm
 )
 {
-    if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
+    #ifndef Foam_Pstream_scatter_nobroadcast
-    {
+    Pstream::broadcast(values, comm);
+    #else
     mapCombineScatter
     (
-            UPstream::linearCommunication(comm),
+        UPstream::whichCommunication(comm),
-            Values,
+        values,
         tag,
         comm
     );
-    }
+    #endif
-    else
-    {
-        mapCombineScatter
-        (
-            UPstream::treeCommunication(comm),
-            Values,
-            tag,
-            comm
-        );
-    }
 }
 
 

src/OpenFOAM/db/IOstreams/Pstreams/PstreamCombineReduceOps.H

@@ -6,6 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2013 OpenFOAM Foundation
+    Copyright (C) 2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -32,11 +33,10 @@ Description
     combined using the given combination function and the result is
     broadcast to all nodes
 
-
 \*---------------------------------------------------------------------------*/
 
-#ifndef PstreamCombineReduceOps_H
+#ifndef Foam_PstreamCombineReduceOps_H
-#define PstreamCombineReduceOps_H
+#define Foam_PstreamCombineReduceOps_H
 
 #include "UPstream.H"
 #include "Pstream.H"
@@ -53,61 +53,32 @@ template<class T, class CombineOp>
 void combineReduce
 (
     const List<UPstream::commsStruct>& comms,
-    T& Value,
+    T& value,
     const CombineOp& cop,
     const int tag,
     const label comm
 )
 {
-    Pstream::combineGather(comms, Value, cop, tag, comm);
+    Pstream::combineGather(comms, value, cop, tag, comm);
-    Pstream::combineScatter(comms, Value, tag, comm);
+    Pstream::broadcast(value, comm);
 }
 
 
 template<class T, class CombineOp>
 void combineReduce
 (
-    T& Value,
+    T& value,
     const CombineOp& cop,
-    const int tag = Pstream::msgType(),
+    const int tag = UPstream::msgType(),
-    const label comm = Pstream::worldComm
+    const label comm = UPstream::worldComm
 )
 {
-    if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
+    if (UPstream::parRun() && UPstream::nProcs(comm) > 1)
     {
-        Pstream::combineGather
+        const auto& comms = UPstream::whichCommunication(comm);
-        (
+
-            UPstream::linearCommunication(comm),
+        Pstream::combineGather(comms, value, cop, tag, comm);
-            Value,
+        Pstream::broadcast(value, comm);
-            cop,
-            tag,
-            comm
-        );
-        Pstream::combineScatter
-        (
-            UPstream::linearCommunication(comm),
-            Value,
-            tag,
-            comm
-        );
-    }
-    else
-    {
-        Pstream::combineGather
-        (
-            UPstream::treeCommunication(comm),
-            Value,
-            cop,
-            tag,
-            comm
-        );
-        Pstream::combineScatter
-        (
-            UPstream::treeCommunication(comm),
-            Value,
-            tag,
-            comm
-        );
     }
 }
 

src/OpenFOAM/db/IOstreams/Pstreams/PstreamExchange.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2016 OpenFOAM Foundation
-    Copyright (C) 2016-2021 OpenCFD Ltd.
+    Copyright (C) 2016-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -43,7 +43,7 @@ void Foam::Pstream::exchangeContainer
     List<Container>& recvBufs,
     const int tag,
     const label comm,
-    const bool block
+    const bool wait
 )
 {
     const label startOfRequests = Pstream::nRequests();
@@ -91,7 +91,7 @@ void Foam::Pstream::exchangeContainer
                 FatalErrorInFunction
                     << "Cannot send outgoing message. "
                     << "to:" << proci << " nBytes:"
-                    << label(sendBufs[proci].size()*sizeof(T))
+                    << label(sendBufs[proci].size_bytes())
                     << Foam::abort(FatalError);
             }
         }
@@ -101,9 +101,9 @@ void Foam::Pstream::exchangeContainer
     // Wait for all to finish
     // ~~~~~~~~~~~~~~~~~~~~~~
 
-    if (block)
+    if (wait)
     {
-        Pstream::waitRequests(startOfRequests);
+        UPstream::waitRequests(startOfRequests);
     }
 }
 
@@ -117,7 +117,7 @@ void Foam::Pstream::exchangeBuf
     List<char*>& recvBufs,
     const int tag,
     const label comm,
-    const bool block
+    const bool wait
 )
 {
     const label startOfRequests = Pstream::nRequests();
@@ -175,9 +175,9 @@ void Foam::Pstream::exchangeBuf
     // Wait for all to finish
     // ~~~~~~~~~~~~~~~~~~~~~~
 
-    if (block)
+    if (wait)
     {
-        Pstream::waitRequests(startOfRequests);
+        UPstream::waitRequests(startOfRequests);
     }
 }
 
@@ -190,7 +190,7 @@ void Foam::Pstream::exchange
     List<Container>& recvBufs,
     const int tag,
     const label comm,
-    const bool block
+    const bool wait
 )
 {
     // OR  static_assert(is_contiguous<T>::value, "Contiguous data only!")
@@ -209,7 +209,7 @@ void Foam::Pstream::exchange
             << Foam::abort(FatalError);
     }
 
-    recvBufs.setSize(sendBufs.size());
+    recvBufs.resize_nocopy(sendBufs.size());
 
     if (UPstream::parRun() && UPstream::nProcs(comm) > 1)
     {
@@ -220,11 +220,11 @@ void Foam::Pstream::exchange
 
             if (proci != Pstream::myProcNo(comm) && nRecv > 0)
             {
-                recvBufs[proci].setSize(nRecv);
+                recvBufs[proci].resize_nocopy(nRecv);
             }
         }
 
-        if (Pstream::maxCommsSize <= 0)
+        if (UPstream::maxCommsSize <= int(sizeof(T)))
         {
             // Do the exchanging in one go
             exchangeContainer<Container, T>
@@ -234,7 +234,7 @@ void Foam::Pstream::exchange
                 recvBufs,
                 tag,
                 comm,
-                block
+                wait
             );
         }
         else
@@ -244,55 +244,64 @@ void Foam::Pstream::exchange
             // guaranteed that some processor's sending size is some other
             // processor's receive size. Also we can ignore any local comms.
 
-            label maxNSend = 0;
+            // We need to send bytes so the number of iterations:
+            //  maxChunkSize                        iterations
+            //  ------------                        ----------
+            //  0                                   0
+            //  1..maxChunkSize                     1
+            //  maxChunkSize+1..2*maxChunkSize      2
+            //      etc.
+
+            const label maxChunkSize(UPstream::maxCommsSize/sizeof(T));
+
+            label nIter(0);
+            {
+                label nSendMax = 0;
                 forAll(sendBufs, proci)
                 {
                     if (proci != Pstream::myProcNo(comm))
                     {
-                    maxNSend = max(maxNSend, sendBufs[proci].size());
+                        nSendMax = max(nSendMax, sendBufs[proci].size());
                     }
                 }
 
-            const label maxNBytes = sizeof(T)*maxNSend;
+                if (nSendMax)
-
-            // We need to send maxNBytes bytes so the number of iterations:
-            //  maxNBytes                           iterations
-            //  ---------                           ----------
-            //  0                                   0
-            //  1..maxCommsSize                     1
-            //  maxCommsSize+1..2*maxCommsSize      2
-            //      etc.
-
-            label nIter;
-            if (maxNBytes == 0)
                 {
-                nIter = 0;
+                    nIter = 1 + ((nSendMax-1)/maxChunkSize);
-            }
-            else
-            {
-                nIter = (maxNBytes-1)/Pstream::maxCommsSize+1;
                 }
                 reduce(nIter, maxOp<label>(), tag, comm);
 
-
+                /// Info<< "send " << nSendMax << " elements ("
-            List<const char*> charSendBufs(sendBufs.size());
+                ///     << (nSendMax*sizeof(T)) << " bytes) in " << nIter
-            List<char*> charRecvBufs(sendBufs.size());
+                ///     << " iterations of " << maxChunkSize << " chunks ("
+                ///     << (maxChunkSize*sizeof(T)) << " bytes) maxCommsSize:"
+                ///     << Pstream::maxCommsSize << endl;
+            }
 
             labelList nRecv(sendBufs.size());
-            labelList startRecv(sendBufs.size(), Zero);
             labelList nSend(sendBufs.size());
+            labelList startRecv(sendBufs.size(), Zero);
             labelList startSend(sendBufs.size(), Zero);
 
-            for (label iter = 0; iter < nIter; iter++)
+            List<const char*> charPtrSend(sendBufs.size());
+            List<char*> charPtrRecv(sendBufs.size());
+
+            for (label iter = 0; iter < nIter; ++iter)
             {
                 forAll(sendBufs, proci)
                 {
                     nSend[proci] = min
                     (
-                        Pstream::maxCommsSize,
+                        maxChunkSize,
                         sendBufs[proci].size()-startSend[proci]
                     );
-                    charSendBufs[proci] =
+                    nRecv[proci] = min
+                    (
+                        maxChunkSize,
+                        recvBufs[proci].size()-startRecv[proci]
+                    );
+
+                    charPtrSend[proci] =
                     (
                         nSend[proci] > 0
                       ? reinterpret_cast<const char*>
@@ -301,14 +310,7 @@ void Foam::Pstream::exchange
                         )
                       : nullptr
                     );
-
+                    charPtrRecv[proci] =
-                    nRecv[proci] = min
-                    (
-                        Pstream::maxCommsSize,
-                        recvBufs[proci].size()-startRecv[proci]
-                    );
-
-                    charRecvBufs[proci] =
                     (
                         nRecv[proci] > 0
                       ? reinterpret_cast<char*>
@@ -319,15 +321,19 @@ void Foam::Pstream::exchange
                     );
                 }
 
+                /// Info<< "iter " << iter
+                ///     << ": beg=" << flatOutput(startSend)
+                ///     << " len=" << flatOutput(nSend) << endl;
+
                 exchangeBuf<T>
                 (
                     nSend,
-                    charSendBufs,
+                    charPtrSend,
                     nRecv,
-                    charRecvBufs,
+                    charPtrRecv,
                     tag,
                     comm,
-                    block
+                    wait
                 );
 
                 forAll(nSend, proci)
@@ -344,6 +350,83 @@ void Foam::Pstream::exchange
 }
 
 
+template<class Container>
+void Foam::Pstream::exchangeSizes
+(
+    const labelUList& sendProcs,
+    const labelUList& recvProcs,
+    const Container& sendBufs,
+    labelList& recvSizes,
+    const label tag,
+    const label comm
+)
+{
+    if (sendBufs.size() != UPstream::nProcs(comm))
+    {
+        FatalErrorInFunction
+            << "Size of container " << sendBufs.size()
+            << " does not equal the number of processors "
+            << UPstream::nProcs(comm)
+            << Foam::abort(FatalError);
+    }
+
+    labelList sendSizes(sendProcs.size());
+    forAll(sendProcs, i)
+    {
+        sendSizes[i] = sendBufs[sendProcs[i]].size();
+    }
+
+    recvSizes.resize_nocopy(sendBufs.size());
+    recvSizes = 0;  // Ensure non-received entries are properly zeroed
+
+    const label startOfRequests = UPstream::nRequests();
+
+    for (const label proci : recvProcs)
+    {
+        UIPstream::read
+        (
+            UPstream::commsTypes::nonBlocking,
+            proci,
+            reinterpret_cast<char*>(&recvSizes[proci]),
+            sizeof(label),
+            tag,
+            comm
+        );
+    }
+
+    forAll(sendProcs, i)
+    {
+        UOPstream::write
+        (
+            UPstream::commsTypes::nonBlocking,
+            sendProcs[i],
+            reinterpret_cast<char*>(&sendSizes[i]),
+            sizeof(label),
+            tag,
+            comm
+        );
+    }
+
+    UPstream::waitRequests(startOfRequests);
+}
+
+
+/// FUTURE?
+///
+/// template<class Container>
+/// void Foam::Pstream::exchangeSizes
+/// (
+///     const labelUList& neighProcs,
+///     const Container& sendBufs,
+///     labelList& recvSizes,
+///     const label tag,
+///     const label comm
+/// )
+/// {
+///     exchangeSizes<Container>(neighProcs, neighProcs, sendBufs, tag, comm);
+/// }
+
+
 template<class Container>
 void Foam::Pstream::exchangeSizes
 (
@@ -366,8 +449,8 @@ void Foam::Pstream::exchangeSizes
     {
         sendSizes[proci] = sendBufs[proci].size();
     }
-    recvSizes.setSize(sendSizes.size());
+    recvSizes.resize_nocopy(sendSizes.size());
-    allToAll(sendSizes, recvSizes, comm);
+    UPstream::allToAll(sendSizes, recvSizes, comm);
 }
 
 
@@ -378,13 +461,13 @@ void Foam::Pstream::exchange
     List<Container>& recvBufs,
     const int tag,
     const label comm,
-    const bool block
+    const bool wait
 )
 {
     labelList recvSizes;
     exchangeSizes(sendBufs, recvSizes, comm);
 
-    exchange<Container, T>(sendBufs, recvSizes, recvBufs, tag, comm, block);
+    exchange<Container, T>(sendBufs, recvSizes, recvBufs, tag, comm, wait);
 }
 
 

src/OpenFOAM/db/IOstreams/Pstreams/PstreamGather.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2017 OpenFOAM Foundation
-    Copyright (C) 2019 OpenCFD Ltd.
+    Copyright (C) 2019-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -32,24 +32,17 @@ Description
 
 \*---------------------------------------------------------------------------*/
 
-#include "UOPstream.H"
 #include "OPstream.H"
-#include "UIPstream.H"
 #include "IPstream.H"
 #include "contiguous.H"
 
-// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-
-namespace Foam
-{
-
 // * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
 
 template<class T, class BinaryOp>
-void Pstream::gather
+void Foam::Pstream::gather
 (
     const List<UPstream::commsStruct>& comms,
-    T& Value,
+    T& value,
     const BinaryOp& bop,
     const int tag,
     const label comm
@@ -57,21 +50,21 @@ void Pstream::gather
 {
     if (UPstream::parRun() && UPstream::nProcs(comm) > 1)
     {
-        // Get my communication order
+        // My communication order
         const commsStruct& myComm = comms[UPstream::myProcNo(comm)];
 
         // Receive from my downstairs neighbours
-        forAll(myComm.below(), belowI)
+        for (const label belowID : myComm.below())
         {
-            T value;
+            T received;
 
             if (is_contiguous<T>::value)
             {
                 UIPstream::read
                 (
                     UPstream::commsTypes::scheduled,
-                    myComm.below()[belowI],
+                    belowID,
-                    reinterpret_cast<char*>(&value),
+                    reinterpret_cast<char*>(&received),
                     sizeof(T),
                     tag,
                     comm
@@ -82,18 +75,18 @@ void Pstream::gather
                 IPstream fromBelow
                 (
                     UPstream::commsTypes::scheduled,
-                    myComm.below()[belowI],
+                    belowID,
                     0,
                     tag,
                     comm
                 );
-                fromBelow >> value;
+                fromBelow >> received;
             }
 
-            Value = bop(Value, value);
+            value = bop(value, received);
         }
 
-        // Send up Value
+        // Send up value
         if (myComm.above() != -1)
         {
             if (is_contiguous<T>::value)
@@ -102,7 +95,7 @@ void Pstream::gather
                 (
                     UPstream::commsTypes::scheduled,
                     myComm.above(),
-                    reinterpret_cast<const char*>(&Value),
+                    reinterpret_cast<const char*>(&value),
                     sizeof(T),
                     tag,
                     comm
@@ -118,7 +111,7 @@ void Pstream::gather
                     tag,
                     comm
                 );
-                toAbove << Value;
+                toAbove << value;
             }
         }
     }
@@ -126,37 +119,33 @@ void Pstream::gather
 
 
 template<class T, class BinaryOp>
-void Pstream::gather
+void Foam::Pstream::gather
 (
-    T& Value,
+    T& value,
     const BinaryOp& bop,
     const int tag,
     const label comm
 )
 {
-    if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
+    gather(UPstream::whichCommunication(comm), value, bop, tag, comm);
-    {
-        gather(UPstream::linearCommunication(comm), Value, bop, tag, comm);
-    }
-    else
-    {
-        gather(UPstream::treeCommunication(comm), Value, bop, tag, comm);
-    }
 }
 
 
 template<class T>
-void Pstream::scatter
+void Foam::Pstream::scatter
 (
     const List<UPstream::commsStruct>& comms,
-    T& Value,
+    T& value,
     const int tag,
     const label comm
 )
 {
+    #ifndef Foam_Pstream_scatter_nobroadcast
+    Pstream::broadcast(value, comm);
+    #else
     if (UPstream::parRun() && UPstream::nProcs(comm) > 1)
     {
-        // Get my communication order
+        // My communication order
         const commsStruct& myComm = comms[UPstream::myProcNo(comm)];
 
         // Receive from up
@@ -168,7 +157,7 @@ void Pstream::scatter
                 (
                     UPstream::commsTypes::scheduled,
                     myComm.above(),
-                    reinterpret_cast<char*>(&Value),
+                    reinterpret_cast<char*>(&value),
                     sizeof(T),
                     tag,
                     comm
@@ -184,7 +173,7 @@ void Pstream::scatter
                     tag,
                     comm
                 );
-                fromAbove >> Value;
+                fromAbove >> value;
             }
         }
 
@@ -193,13 +182,15 @@ void Pstream::scatter
         // (only when using a tree schedule!) first.
         forAllReverse(myComm.below(), belowI)
         {
+            const label belowID = myComm.below()[belowI];
+
             if (is_contiguous<T>::value)
             {
                 UOPstream::write
                 (
                     UPstream::commsTypes::scheduled,
-                    myComm.below()[belowI],
+                    belowID,
-                    reinterpret_cast<const char*>(&Value),
+                    reinterpret_cast<const char*>(&value),
                     sizeof(T),
                     tag,
                     comm
@@ -210,34 +201,28 @@ void Pstream::scatter
                 OPstream toBelow
                 (
                     UPstream::commsTypes::scheduled,
-                    myComm.below()[belowI],
+                    belowID,
                     0,
                     tag,
                     comm
                 );
-                toBelow << Value;
+                toBelow << value;
             }
         }
     }
+    #endif
 }
 
 
 template<class T>
-void Pstream::scatter(T& Value, const int tag, const label comm)
+void Foam::Pstream::scatter(T& value, const int tag, const label comm)
 {
-    if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
+    #ifndef Foam_Pstream_scatter_nobroadcast
-    {
+    Pstream::broadcast(value, comm);
-        scatter(UPstream::linearCommunication(comm), Value, tag, comm);
+    #else
-    }
+    scatter(UPstream::whichCommunication(comm), value, tag, comm);
-    else
+    #endif
-    {
-        scatter(UPstream::treeCommunication(comm), Value, tag, comm);
-    }
 }
 
 
-// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-
-} // End namespace Foam
-
 // ************************************************************************* //

src/OpenFOAM/db/IOstreams/Pstreams/PstreamGatherList.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2017 OpenFOAM Foundation
-    Copyright (C) 2015-2021 OpenCFD Ltd.
+    Copyright (C) 2015-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -29,7 +29,7 @@ Description
     communication schedule (usually linear-to-master or tree-to-master).
     The gathered data will be a list with element procID the data from processor
     procID. Before calling every processor should insert its value into
-    Values[UPstream::myProcNo(comm)].
+    values[UPstream::myProcNo(comm)].
     Note: after gather every processor only knows its own data and that of the
     processors below it. Only the 'master' of the communication schedule holds
     a fully filled List. Use scatter to distribute the data.
@@ -40,61 +40,55 @@ Description
 #include "OPstream.H"
 #include "contiguous.H"
 
-// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-
-namespace Foam
-{
-
 // * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
 
 template<class T>
-void Pstream::gatherList
+void Foam::Pstream::gatherList
 (
     const List<UPstream::commsStruct>& comms,
-    List<T>& Values,
+    List<T>& values,
     const int tag,
     const label comm
 )
 {
     if (UPstream::parRun() && UPstream::nProcs(comm) > 1)
     {
-        if (Values.size() != UPstream::nProcs(comm))
+        if (values.size() != UPstream::nProcs(comm))
         {
             FatalErrorInFunction
-                << "Size of list:" << Values.size()
+                << "Size of list:" << values.size()
                 << " does not equal the number of processors:"
                 << UPstream::nProcs(comm)
                 << Foam::abort(FatalError);
         }
 
-        // Get my communication order
+        // My communication order
         const commsStruct& myComm = comms[UPstream::myProcNo(comm)];
 
         // Receive from my downstairs neighbours
-        forAll(myComm.below(), belowI)
+        for (const label belowID : myComm.below())
         {
-            label belowID = myComm.below()[belowI];
             const labelList& belowLeaves = comms[belowID].allBelow();
 
             if (is_contiguous<T>::value)
             {
-                List<T> receivedValues(belowLeaves.size() + 1);
+                List<T> received(belowLeaves.size() + 1);
 
                 UIPstream::read
                 (
                     UPstream::commsTypes::scheduled,
                     belowID,
-                    receivedValues.data_bytes(),
+                    received.data_bytes(),
-                    receivedValues.size_bytes(),
+                    received.size_bytes(),
                     tag,
                     comm
                 );
 
-                Values[belowID] = receivedValues[0];
+                values[belowID] = received[0];
 
                 forAll(belowLeaves, leafI)
                 {
-                    Values[belowLeaves[leafI]] = receivedValues[leafI + 1];
+                    values[belowLeaves[leafI]] = received[leafI + 1];
                 }
             }
             else
@@ -107,32 +101,31 @@ void Pstream::gatherList
                     tag,
                     comm
                 );
-                fromBelow >> Values[belowID];
+                fromBelow >> values[belowID];
 
                 if (debug & 2)
                 {
                     Pout<< " received through "
                         << belowID << " data from:" << belowID
-                        << " data:" << Values[belowID] << endl;
+                        << " data:" << values[belowID] << endl;
                 }
 
                 // Receive from all other processors below belowID
-                forAll(belowLeaves, leafI)
+                for (const label leafID : belowLeaves)
                 {
-                    label leafID = belowLeaves[leafI];
+                    fromBelow >> values[leafID];
-                    fromBelow >> Values[leafID];
 
                     if (debug & 2)
                     {
                         Pout<< " received through "
                             << belowID << " data from:" << leafID
-                            << " data:" << Values[leafID] << endl;
+                            << " data:" << values[leafID] << endl;
                     }
                 }
             }
         }
 
-        // Send up from Values:
+        // Send up from values:
         // - my own value first
         // - all belowLeaves next
         if (myComm.above() != -1)
@@ -143,25 +136,25 @@ void Pstream::gatherList
             {
                 Pout<< " sending to " << myComm.above()
                     << " data from me:" << UPstream::myProcNo(comm)
-                    << " data:" << Values[UPstream::myProcNo(comm)] << endl;
+                    << " data:" << values[UPstream::myProcNo(comm)] << endl;
             }
 
             if (is_contiguous<T>::value)
             {
-                List<T> sendingValues(belowLeaves.size() + 1);
+                List<T> sending(belowLeaves.size() + 1);
-                sendingValues[0] = Values[UPstream::myProcNo(comm)];
+                sending[0] = values[UPstream::myProcNo(comm)];
 
                 forAll(belowLeaves, leafI)
                 {
-                    sendingValues[leafI + 1] = Values[belowLeaves[leafI]];
+                    sending[leafI + 1] = values[belowLeaves[leafI]];
                 }
 
-                OPstream::write
+                UOPstream::write
                 (
                     UPstream::commsTypes::scheduled,
                     myComm.above(),
-                    sendingValues.cdata_bytes(),
+                    sending.cdata_bytes(),
-                    sendingValues.size_bytes(),
+                    sending.size_bytes(),
                     tag,
                     comm
                 );
@@ -176,19 +169,17 @@ void Pstream::gatherList
                     tag,
                     comm
                 );
-                toAbove << Values[UPstream::myProcNo(comm)];
+                toAbove << values[UPstream::myProcNo(comm)];
 
-                forAll(belowLeaves, leafI)
+                for (const label leafID : belowLeaves)
                 {
-                    label leafID = belowLeaves[leafI];
-
                     if (debug & 2)
                     {
                         Pout<< " sending to "
                             << myComm.above() << " data from:" << leafID
-                            << " data:" << Values[leafID] << endl;
+                            << " data:" << values[leafID] << endl;
                     }
-                    toAbove << Values[leafID];
+                    toAbove << values[leafID];
                 }
             }
         }
@@ -197,40 +188,37 @@ void Pstream::gatherList
 
 
 template<class T>
-void Pstream::gatherList(List<T>& Values, const int tag, const label comm)
+void Foam::Pstream::gatherList(List<T>& values, const int tag, const label comm)
 {
-    if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
+    gatherList(UPstream::whichCommunication(comm), values, tag, comm);
-    {
-        gatherList(UPstream::linearCommunication(comm), Values, tag, comm);
-    }
-    else
-    {
-        gatherList(UPstream::treeCommunication(comm), Values, tag, comm);
-    }
 }
 
 
 template<class T>
-void Pstream::scatterList
+void Foam::Pstream::scatterList
 (
     const List<UPstream::commsStruct>& comms,
-    List<T>& Values,
+    List<T>& values,
     const int tag,
     const label comm
 )
 {
+    // Apart from the additional (size == nProcs) check, the only difference
+    // between scatterList() and using broadcast(List<T>&) or a regular
+    // scatter(List<T>&) is that processor-local data is skipped.
+
     if (UPstream::parRun() && UPstream::nProcs(comm) > 1)
     {
-        if (Values.size() != UPstream::nProcs(comm))
+        if (values.size() != UPstream::nProcs(comm))
         {
             FatalErrorInFunction
-                << "Size of list:" << Values.size()
+                << "Size of list:" << values.size()
                 << " does not equal the number of processors:"
                 << UPstream::nProcs(comm)
                 << Foam::abort(FatalError);
         }
 
-        // Get my communication order
+        // My communication order
         const commsStruct& myComm = comms[UPstream::myProcNo(comm)];
 
         // Receive from up
@@ -240,21 +228,21 @@ void Pstream::scatterList
 
             if (is_contiguous<T>::value)
             {
-                List<T> receivedValues(notBelowLeaves.size());
+                List<T> received(notBelowLeaves.size());
 
                 UIPstream::read
                 (
                     UPstream::commsTypes::scheduled,
                     myComm.above(),
-                    receivedValues.data_bytes(),
+                    received.data_bytes(),
-                    receivedValues.size_bytes(),
+                    received.size_bytes(),
                     tag,
                     comm
                 );
 
                 forAll(notBelowLeaves, leafI)
                 {
-                    Values[notBelowLeaves[leafI]] = receivedValues[leafI];
+                    values[notBelowLeaves[leafI]] = received[leafI];
                 }
             }
             else
@@ -268,16 +256,15 @@ void Pstream::scatterList
                     comm
                 );
 
-                forAll(notBelowLeaves, leafI)
+                for (const label leafID : notBelowLeaves)
                 {
-                    label leafID = notBelowLeaves[leafI];
+                    fromAbove >> values[leafID];
-                    fromAbove >> Values[leafID];
 
                     if (debug)
                     {
                         Pout<< " received through "
                             << myComm.above() << " data for:" << leafID
-                            << " data:" << Values[leafID] << endl;
+                            << " data:" << values[leafID] << endl;
                     }
                 }
             }
@@ -286,24 +273,24 @@ void Pstream::scatterList
         // Send to my downstairs neighbours
         forAllReverse(myComm.below(), belowI)
         {
-            label belowID = myComm.below()[belowI];
+            const label belowID = myComm.below()[belowI];
             const labelList& notBelowLeaves = comms[belowID].allNotBelow();
 
             if (is_contiguous<T>::value)
             {
-                List<T> sendingValues(notBelowLeaves.size());
+                List<T> sending(notBelowLeaves.size());
 
                 forAll(notBelowLeaves, leafI)
                 {
-                    sendingValues[leafI] = Values[notBelowLeaves[leafI]];
+                    sending[leafI] = values[notBelowLeaves[leafI]];
                 }
 
-                OPstream::write
+                UOPstream::write
                 (
                     UPstream::commsTypes::scheduled,
                     belowID,
-                    sendingValues.cdata_bytes(),
+                    sending.cdata_bytes(),
-                    sendingValues.size_bytes(),
+                    sending.size_bytes(),
                     tag,
                     comm
                 );
@@ -320,16 +307,15 @@ void Pstream::scatterList
                 );
 
                 // Send data destined for all other processors below belowID
-                forAll(notBelowLeaves, leafI)
+                for (const label leafID : notBelowLeaves)
                 {
-                    label leafID = notBelowLeaves[leafI];
+                    toBelow << values[leafID];
-                    toBelow << Values[leafID];
 
                     if (debug)
                     {
                         Pout<< " sent through "
                             << belowID << " data for:" << leafID
-                            << " data:" << Values[leafID] << endl;
+                            << " data:" << values[leafID] << endl;
                     }
                 }
             }
@@ -339,21 +325,15 @@ void Pstream::scatterList
 
 
 template<class T>
-void Pstream::scatterList(List<T>& Values, const int tag, const label comm)
+void Foam::Pstream::scatterList
+(
+    List<T>& values,
+    const int tag,
+    const label comm
+)
 {
-    if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
+    scatterList(UPstream::whichCommunication(comm), values, tag, comm);
-    {
-        scatterList(UPstream::linearCommunication(comm), Values, tag, comm);
-    }
-    else
-    {
-        scatterList(UPstream::treeCommunication(comm), Values, tag, comm);
-    }
 }
 
 
-// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-
-} // End namespace Foam
-
 // ************************************************************************* //

src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2016 OpenFOAM Foundation
-    Copyright (C) 2016 OpenCFD Ltd.
+    Copyright (C) 2016-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -32,11 +32,11 @@ Description
 
 \*---------------------------------------------------------------------------*/
 
-#ifndef PstreamReduceOps_H
+#ifndef Foam_PstreamReduceOps_H
-#define PstreamReduceOps_H
+#define Foam_PstreamReduceOps_H
 
 #include "ops.H"
-#include "vector2D.H"
+#include "FixedList.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
@@ -45,12 +45,12 @@ namespace Foam
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
-// Reduce operation with user specified communication schedule
+//- Reduce operation with user specified communication schedule
 template<class T, class BinaryOp>
 void reduce
 (
     const List<UPstream::commsStruct>& comms,
-    T& Value,
+    T& value,
     const BinaryOp& bop,
     const int tag,
     const label comm
@@ -58,96 +58,85 @@ void reduce
 {
     if (UPstream::warnComm != -1 && comm != UPstream::warnComm)
     {
-        Pout<< "** reducing:" << Value << " with comm:" << comm
+        Pout<< "** reducing:" << value << " with comm:" << comm << endl;
-            << endl;
         error::printStack(Pout);
     }
-    Pstream::gather(comms, Value, bop, tag, comm);
+    Pstream::gather(comms, value, bop, tag, comm);
-    Pstream::scatter(comms, Value, tag, comm);
+    Pstream::broadcast(value, comm);
 }
 
 
-// Reduce using either linear or tree communication schedule
+//- Reduce (inplace) using either linear or tree communication schedule
 template<class T, class BinaryOp>
 void reduce
 (
-    T& Value,
+    T& value,
     const BinaryOp& bop,
-    const int tag = Pstream::msgType(),
+    const int tag = UPstream::msgType(),
     const label comm = UPstream::worldComm
 )
 {
-    if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
+    if (UPstream::parRun())
     {
-        reduce(UPstream::linearCommunication(comm), Value, bop, tag, comm);
+        reduce(UPstream::whichCommunication(comm), value, bop, tag, comm);
-    }
-    else
-    {
-        reduce(UPstream::treeCommunication(comm), Value, bop, tag, comm);
     }
 }
 
 
-// Reduce using either linear or tree communication schedule
+//- Reduce (copy) and return value
 template<class T, class BinaryOp>
 T returnReduce
 (
-    const T& Value,
+    const T& value,
     const BinaryOp& bop,
-    const int tag = Pstream::msgType(),
+    const int tag = UPstream::msgType(),
     const label comm = UPstream::worldComm
 )
 {
-    T WorkValue(Value);
+    T work(value);
-
+    reduce(work, bop, tag, comm);
-    if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
+    return work;
-    {
-        reduce
-        (
-            UPstream::linearCommunication(comm),
-            WorkValue,
-            bop,
-            tag,
-            comm
-        );
-    }
-    else
-    {
-        reduce
-        (
-            UPstream::treeCommunication(comm),
-            WorkValue,
-            bop,
-            tag,
-            comm
-        );
-    }
-
-    return WorkValue;
 }
 
 
-// Reduce with sum of both value and count (for averaging)
+//- Reduce with sum of both value and count (for averaging)
 template<class T>
 void sumReduce
 (
-    T& Value,
+    T& value,
-    label& Count,
+    label& count,
-    const int tag = Pstream::msgType(),
+    const int tag = UPstream::msgType(),
     const label comm = UPstream::worldComm
 )
 {
-    reduce(Value, sumOp<T>(), tag, comm);
+    if (UPstream::parRun())
-    reduce(Count, sumOp<label>(), tag, comm);
+    {
+        reduce(value, sumOp<T>(), tag, comm);
+        reduce(count, sumOp<label>(), tag, comm);
+    }
 }
 
 
-// Non-blocking version of reduce. Sets request.
+//- Reduce multiple values (identical size on all processes!)
+template<class T, class BinaryOp>
+void reduce
+(
+    T values[],
+    const int size,
+    const BinaryOp&,
+    const int tag,
+    const label comm
+)
+{
+    NotImplemented;
+}
+
+//- Non-blocking reduce single value. Sets request.
 template<class T, class BinaryOp>
 void reduce
 (
     T& Value,
-    const BinaryOp& bop,
+    const BinaryOp&,
     const int tag,
     const label comm,
     label& request
@@ -156,13 +145,14 @@ void reduce
     NotImplemented;
 }
 
-// Non-blocking version of reduce. Sets request.
+//- Non-blocking reduce multiple values (identical size on all processes!)
+//- Sets request.
 template<class T, class BinaryOp>
 void reduce
 (
-    T Value[],
+    T values[],
     const int size,
-    const BinaryOp& bop,
+    const BinaryOp&,
     const int tag,
     const label comm,
     label& request
@@ -172,111 +162,139 @@ void reduce
 }
 
 
-// Insist there are specialisations for the common reductions of scalar(s)
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+// Specialisations for bool
+
+//- Logical (and) reduction
 void reduce
 (
-    scalar& Value,
+    bool& value,
-    const sumOp<scalar>& bop,
+    const andOp<bool>&,
-    const int tag = Pstream::msgType(),
+    const int tag = UPstream::msgType(),  /*!< (ignored) */
     const label comm = UPstream::worldComm
 );
 
+//- Logical (or) reduction
 void reduce
 (
-    scalar& Value,
+    bool& value,
-    const minOp<scalar>& bop,
+    const orOp<bool>&,
-    const int tag = Pstream::msgType(),
+    const int tag = UPstream::msgType(),  /*!< (ignored) */
     const label comm = UPstream::worldComm
 );
 
-void reduce
-(
-    vector2D& Value,
-    const sumOp<vector2D>& bop,
-    const int tag = Pstream::msgType(),
-    const label comm = UPstream::worldComm
-);
 
-void sumReduce
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-(
-    scalar& Value,
-    label& Count,
-    const int tag = Pstream::msgType(),
-    const label comm = UPstream::worldComm
-);
 
-void reduce
+// Specialisations for common reduction types
-(
-    scalar& Value,
-    const sumOp<scalar>& bop,
-    const int tag,
-    const label comm,
-    label& request
-);
 
-void reduce
+#undef  Pstream_CommonReductions
-(
+#define Pstream_CommonReductions(Native)                                      \
-    scalar Value[],
+                                                                              \
-    const int size,
+/*! \brief Reduce (min) single Native value */                                \
-    const sumOp<scalar>& bop,
+void reduce                                                                   \
-    const int tag,
+(                                                                             \
-    const label comm,
+    Native& value,                                                            \
-    label& request
+    const minOp<Native>&,                                                     \
+    const int tag = UPstream::msgType(),  /*!< (ignored) */                   \
+    const label comm = UPstream::worldComm                                    \
+);                                                                            \
+                                                                              \
+/*! \brief Reduce (max) single Native value */                                \
+void reduce                                                                   \
+(                                                                             \
+    Native& value,                                                            \
+    const maxOp<Native>&,                                                     \
+    const int tag = UPstream::msgType(),  /*!< (ignored) */                   \
+    const label comm = UPstream::worldComm                                    \
+);                                                                            \
+                                                                              \
+/*! \brief Reduce (sum) single Native value */                                \
+void reduce                                                                   \
+(                                                                             \
+    Native& value,                                                            \
+    const sumOp<Native>&,                                                     \
+    const int tag = UPstream::msgType(),  /*!< (ignored) */                   \
+    const label comm = UPstream::worldComm                                    \
+);                                                                            \
+                                                                              \
+/*! \brief Reduce (sum) multiple Native values (identical size all procs!) */ \
+void reduce                                                                   \
+(                                                                             \
+    Native values[],                                                          \
+    const int size,                                                           \
+    const sumOp<Native>&,                                                     \
+    const int tag,  /*!< (ignored) */                                         \
+    const label comm                                                          \
+);                                                                            \
+                                                                              \
+/*! \brief Reduce (sum) multiple Native values */                             \
+template<unsigned N>                                                          \
+inline void reduce                                                            \
+(                                                                             \
+    FixedList<Native, N>& values,                                             \
+    const sumOp<Native>&,                                                     \
+    const int tag = UPstream::msgType(),  /*!< (ignored) */                   \
+    const label comm = UPstream::worldComm                                    \
+)                                                                             \
+{                                                                             \
+    reduce(values.data(), int(values.size()), sumOp<Native>(), tag, comm);    \
+}
+
+
+Pstream_CommonReductions(int32_t);
+Pstream_CommonReductions(int64_t);
+Pstream_CommonReductions(uint32_t);
+Pstream_CommonReductions(uint64_t);
+Pstream_CommonReductions(float);
+Pstream_CommonReductions(double);
+
+#undef Pstream_CommonReductions
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+// Specialisations for floating-point types
+
+#undef  Pstream_FloatReductions
+#define Pstream_FloatReductions(Native)                                       \
+                                                                              \
+/*! \brief Sum of both Native value and count (for averaging) */              \
+void sumReduce                                                                \
+(                                                                             \
+    Native& value,                                                            \
+    label& count,                                                             \
+    const int tag = UPstream::msgType(),  /*!< (ignored) */                   \
+    const label comm = UPstream::worldComm                                    \
+);                                                                            \
+                                                                              \
+/*! \brief Non-blocking reduce (sum) single Native value. Sets request */     \
+void reduce                                                                   \
+(                                                                             \
+    Native& value,                                                            \
+    const sumOp<Native>&,                                                     \
+    const int tag,  /*!< (ignored) */                                         \
+    const label comm,                                                         \
+    label& requestID                                                          \
+);                                                                            \
+                                                                              \
+/*! \brief Non-blocking reduce (sum) multiple Native values. Sets request */  \
+void reduce                                                                   \
+(                                                                             \
+    Native values[],                                                          \
+    const int size,                                                           \
+    const sumOp<Native>&,                                                     \
+    const int tag,  /*!< (ignored) */                                         \
+    const label comm,                                                         \
+    label& requestID                                                          \
 );
 
 
-#if defined(WM_SPDP)
+Pstream_FloatReductions(float);
-void reduce
+Pstream_FloatReductions(double);
-(
-    solveScalar& Value,
-    const sumOp<solveScalar>& bop,
-    const int tag = Pstream::msgType(),
-    const label comm = UPstream::worldComm
-);
 
-void reduce
+#undef Pstream_FloatReductions
-(
-    solveScalar& Value,
-    const minOp<solveScalar>& bop,
-    const int tag = Pstream::msgType(),
-    const label comm = UPstream::worldComm
-);
-
-void reduce
-(
-    Vector2D<solveScalar>& Value,
-    const sumOp<Vector2D<solveScalar>>& bop,
-    const int tag = Pstream::msgType(),
-    const label comm = UPstream::worldComm
-);
-
-void sumReduce
-(
-    solveScalar& Value,
-    label& Count,
-    const int tag = Pstream::msgType(),
-    const label comm = UPstream::worldComm
-);
-
-void reduce
-(
-    solveScalar& Value,
-    const sumOp<solveScalar>& bop,
-    const int tag,
-    const label comm,
-    label& request
-);
-
-void reduce
-(
-    solveScalar Value[],
-    const int size,
-    const sumOp<solveScalar>& bop,
-    const int tag,
-    const label comm,
-    label& request
-);
-#endif
 
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //

src/OpenFOAM/db/IOstreams/Pstreams/UIPstream.H

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2013 OpenFOAM Foundation
-    Copyright (C) 2017-2021 OpenCFD Ltd.
+    Copyright (C) 2017-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -25,24 +25,26 @@ License
     along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
 
 Class
-    Foam::UIPstream
+    Foam::UIPstreamBase
 
 Description
-    Input inter-processor communications stream operating on external
+    Base class for input inter-processor communications stream
-    buffer.
+    (ie, parallel streams).
+    Not to be used directly, thus contructors are protected.
 
 SourceFiles
-    UIPstream.C
+    UIPstreamBase.C
 
 \*---------------------------------------------------------------------------*/
 
 #include "Pstream.H"
 
-#ifndef UIPstream_H
+#ifndef Foam_UIPstream_H
-#define UIPstream_H
+#define Foam_UIPstream_H
 
 #include "UPstream.H"
 #include "Istream.H"
+#include "DynamicList.H"
 #include "PstreamBuffers.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
@@ -51,31 +53,14 @@ namespace Foam
 {
 
 /*---------------------------------------------------------------------------*\
-                           Class UIPstream Declaration
+                        Class UIPstreamBase Declaration
 \*---------------------------------------------------------------------------*/
 
-class UIPstream
+class UIPstreamBase
 :
     public UPstream,
     public Istream
 {
-    // Private Data
-
-        int fromProcNo_;
-
-        DynamicList<char>& recvBuf_;
-
-        label& recvBufPos_;
-
-        const int tag_;
-
-        const label comm_;
-
-        const bool clearAtEnd_;
-
-        int messageSize_;
-
-
     // Private Member Functions
 
         //- Check buffer position against messageSize_ for EOF
@@ -96,14 +81,31 @@ class UIPstream
         inline Istream& readString(std::string& str);
 
 
-public:
+protected:
 
-    // Constructors
+    // Protected Data
+
+        int fromProcNo_;
+
+        DynamicList<char>& recvBuf_;
+
+        label& recvBufPos_;
+
+        const int tag_;
+
+        const label comm_;
+
+        const bool clearAtEnd_;
+
+        int messageSize_;
+
+
+    // Protected Constructors
 
         //- Construct given process index to read from using the given
         //- attached receive buffer, optional communication characteristics
         //- and IO format
-        UIPstream
+        UIPstreamBase
         (
             const commsTypes commsType,
             const int fromProcNo,
@@ -116,11 +118,14 @@ public:
         );
 
         //- Construct given buffers
-        UIPstream(const int fromProcNo, PstreamBuffers& buffers);
+        UIPstreamBase(const int fromProcNo, PstreamBuffers& buffers);
 
 
-    //- Destructor
+public:
-    ~UIPstream();
+
+
+    //- Destructor. Optionally clears external receive buffer.
+    virtual ~UIPstreamBase();
 
 
     // Member Functions
@@ -128,7 +133,7 @@ public:
         // Inquiry
 
             //- Return flags of output stream
-            ios_base::fmtflags flags() const
+            virtual ios_base::fmtflags flags() const
             {
                 return ios_base::fmtflags(0);
             }
@@ -136,18 +141,6 @@ public:
 
         // Read Functions
 
-            //- Read into given buffer from given processor
-            //  \return the message size
-            static label read
-            (
-                const commsTypes commsType,
-                const int fromProcNo,
-                char* buf,
-                const std::streamsize bufSize,
-                const int tag = UPstream::msgType(),
-                const label communicator = UPstream::worldComm
-            );
-
             //- Return next token from stream
             Istream& read(token& t);
 
@@ -184,14 +177,17 @@ public:
                 return true;
             }
 
-            //- Rewind the stream so that it may be read again
+
-            void rewind();
+    // Positioning
+
+        //- Rewind the receive stream position so that it may be read again
+        virtual void rewind();
 
 
     // Edit
 
         //- Set flags of stream
-            ios_base::fmtflags flags(const ios_base::fmtflags)
+        virtual ios_base::fmtflags flags(const ios_base::fmtflags)
         {
             return ios_base::fmtflags(0);
         }
@@ -204,6 +200,133 @@ public:
 };
 
 
+/*---------------------------------------------------------------------------*\
+                           Class UIPstream Declaration
+\*---------------------------------------------------------------------------*/
+
+//- Input inter-processor communications stream
+//- using MPI send/recv etc. - operating on external buffer.
+class UIPstream
+:
+    public UIPstreamBase
+{
+    // Private Member Functions
+
+        //- Initial buffer recv, called by constructor (blocking | scheduled)
+        void bufferIPCrecv();
+
+
+public:
+
+    // Constructors
+
+        //- Construct given process index to read from using the given
+        //- attached receive buffer, optional communication characteristics
+        //- and IO format
+        UIPstream
+        (
+            const commsTypes commsType,
+            const int fromProcNo,
+            DynamicList<char>& receiveBuf,
+            label& receiveBufPosition,
+            const int tag = UPstream::msgType(),
+            const label comm = UPstream::worldComm,
+            const bool clearAtEnd = false,   // destroy receiveBuf if at end
+            IOstreamOption::streamFormat fmt = IOstreamOption::BINARY
+        );
+
+        //- Construct given buffers
+        UIPstream(const int fromProcNo, PstreamBuffers& buffers);
+
+
+    //- Destructor
+    virtual ~UIPstream() = default;
+
+
+    // Member Functions
+
+        //- Use all read methods from base
+        using UIPstreamBase::read;
+
+
+    // Static Functions
+
+        //- Read buffer contents from given processor
+        //  \return the message size
+        static label read
+        (
+            const commsTypes commsType,
+            const int fromProcNo,
+            char* buf,
+            const std::streamsize bufSize,
+            const int tag = UPstream::msgType(),
+            const label comm = UPstream::worldComm
+        );
+};
+
+
+/*---------------------------------------------------------------------------*\
+                         Class UIPBstream Declaration
+\*---------------------------------------------------------------------------*/
+
+//- Input inter-processor communications stream
+//- using MPI broadcast - operating on external buffer.
+class UIPBstream
+:
+    public UIPstreamBase
+{
+    // Private Member Functions
+
+        //- Initial buffer recv via broadcast, called by constructor
+        void bufferIPCrecv();
+
+
+public:
+
+    // Constructors
+
+        //- Construct given process index to read from using the given
+        //- attached receive buffer, optional communication characteristics
+        //- and IO format
+        UIPBstream
+        (
+            const commsTypes commsType,     //!< ignored
+            const int rootProcNo,           //!< normally UPstream::masterNo()
+            DynamicList<char>& receiveBuf,
+            label& receiveBufPosition,
+            const int tag = UPstream::msgType(),  //!< ignored
+            const label comm = UPstream::worldComm,
+            const bool clearAtEnd = false,  //!< destroy receiveBuf if at end
+            IOstreamOption::streamFormat fmt = IOstreamOption::BINARY
+        );
+
+
+    //- Destructor
+    virtual ~UIPBstream() = default;
+
+
+    // Member Functions
+
+        //- Use all read methods from base
+        using UIPstreamBase::read;
+
+
+    // Static Functions
+
+        //- Wrapped version of UPstream::broadcast
+        //  \return the message size
+        static label read
+        (
+            const commsTypes commsTypes,    //!< ignored
+            const int rootProcNo,           //!< normally UPstream::masterNo()
+            char* buf,
+            const std::streamsize bufSize,
+            const int tag = UPstream::msgType(),  //!< ignored
+            const label comm = UPstream::worldComm
+        );
+};
+
+
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
 } // End namespace Foam

src/OpenFOAM/db/IOstreams/Pstreams/UIPstreamBase.C

@@ -74,7 +74,7 @@ inline static label byteAlign(const label pos, const size_t align)
 
 // * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
 
-inline void Foam::UIPstream::checkEof()
+inline void Foam::UIPstreamBase::checkEof()
 {
     if (recvBufPos_ == messageSize_)
     {
@@ -83,14 +83,14 @@ inline void Foam::UIPstream::checkEof()
 }
 
 
-inline void Foam::UIPstream::prepareBuffer(const size_t align)
+inline void Foam::UIPstreamBase::prepareBuffer(const size_t align)
 {
     recvBufPos_ = byteAlign(recvBufPos_, align);
 }
 
 
 template<class T>
-inline void Foam::UIPstream::readFromBuffer(T& val)
+inline void Foam::UIPstreamBase::readFromBuffer(T& val)
 {
     prepareBuffer(sizeof(T));
 
@@ -100,7 +100,7 @@ inline void Foam::UIPstream::readFromBuffer(T& val)
 }
 
 
-inline void Foam::UIPstream::readFromBuffer
+inline void Foam::UIPstreamBase::readFromBuffer
 (
     void* data,
     const size_t count
@@ -119,7 +119,7 @@ inline void Foam::UIPstream::readFromBuffer
 }
 
 
-inline Foam::Istream& Foam::UIPstream::readString(std::string& str)
+inline Foam::Istream& Foam::UIPstreamBase::readString(std::string& str)
 {
     // Use std::string::assign() to copy content, including '\0'.
     // Stripping (when desired) is the responsibility of the sending side.
@@ -142,15 +142,78 @@ inline Foam::Istream& Foam::UIPstream::readString(std::string& str)
 }
 
 
+// * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
+
+Foam::UIPstreamBase::UIPstreamBase
+(
+    const commsTypes commsType,
+    const int fromProcNo,
+    DynamicList<char>& receiveBuf,
+    label& receiveBufPosition,
+    const int tag,
+    const label comm,
+    const bool clearAtEnd,
+    IOstreamOption::streamFormat fmt
+)
+:
+    UPstream(commsType),
+    Istream(fmt, IOstreamOption::currentVersion),
+    fromProcNo_(fromProcNo),
+    recvBuf_(receiveBuf),
+    recvBufPos_(receiveBufPosition),
+    tag_(tag),
+    comm_(comm),
+    clearAtEnd_(clearAtEnd),
+    messageSize_(0)
+{
+    setOpened();
+    setGood();
+}
+
+
+Foam::UIPstreamBase::UIPstreamBase
+(
+    const int fromProcNo,
+    PstreamBuffers& buffers
+)
+:
+    UPstream(buffers.commsType()),
+    Istream(buffers.format(), IOstreamOption::currentVersion),
+    fromProcNo_(fromProcNo),
+    recvBuf_(buffers.recvBuf_[fromProcNo]),
+    recvBufPos_(buffers.recvBufPos_[fromProcNo]),
+    tag_(buffers.tag()),
+    comm_(buffers.comm()),
+    clearAtEnd_(buffers.allowClearRecv()),
+    messageSize_(0)
+{
+    if
+    (
+        commsType() != UPstream::commsTypes::scheduled
+     && !buffers.finished()
+    )
+    {
+        FatalErrorInFunction
+            << "PstreamBuffers::finishedSends() never called." << endl
+            << "Please call PstreamBuffers::finishedSends() after doing"
+            << " all your sends (using UOPstream) and before doing any"
+            << " receives (using UIPstream)" << Foam::exit(FatalError);
+    }
+
+    setOpened();
+    setGood();
+}
+
+
 // * * * * * * * * * * * * * * * * Destructor  * * * * * * * * * * * * * * * //
 
-Foam::UIPstream::~UIPstream()
+Foam::UIPstreamBase::~UIPstreamBase()
 {
     if (clearAtEnd_ && eof())
     {
         if (debug)
         {
-            Pout<< "UIPstream::~UIPstream() : tag:" << tag_
+            Pout<< "UIPstreamBase Destructor : tag:" << tag_
                 << " fromProcNo:" << fromProcNo_
                 << " clearing receive buffer of size "
                 << recvBuf_.size()
@@ -163,7 +226,7 @@ Foam::UIPstream::~UIPstream()
 
 // * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * * //
 
-Foam::Istream& Foam::UIPstream::read(token& t)
+Foam::Istream& Foam::UIPstreamBase::read(token& t)
 {
     // Return the put back token if it exists
     // - with additional handling for special stream flags
@@ -341,7 +404,7 @@ Foam::Istream& Foam::UIPstream::read(token& t)
 }
 
 
-Foam::Istream& Foam::UIPstream::read(char& c)
+Foam::Istream& Foam::UIPstreamBase::read(char& c)
 {
     c = recvBuf_[recvBufPos_];
     ++recvBufPos_;
@@ -350,40 +413,40 @@ Foam::Istream& Foam::UIPstream::read(char& c)
 }
 
 
-Foam::Istream& Foam::UIPstream::read(word& str)
+Foam::Istream& Foam::UIPstreamBase::read(word& str)
 {
     return readString(str);
 }
 
 
-Foam::Istream& Foam::UIPstream::read(string& str)
+Foam::Istream& Foam::UIPstreamBase::read(string& str)
 {
     return readString(str);
 }
 
 
-Foam::Istream& Foam::UIPstream::read(label& val)
+Foam::Istream& Foam::UIPstreamBase::read(label& val)
 {
     readFromBuffer(val);
     return *this;
 }
 
 
-Foam::Istream& Foam::UIPstream::read(floatScalar& val)
+Foam::Istream& Foam::UIPstreamBase::read(floatScalar& val)
 {
     readFromBuffer(val);
     return *this;
 }
 
 
-Foam::Istream& Foam::UIPstream::read(doubleScalar& val)
+Foam::Istream& Foam::UIPstreamBase::read(doubleScalar& val)
 {
     readFromBuffer(val);
     return *this;
 }
 
 
-Foam::Istream& Foam::UIPstream::read(char* data, std::streamsize count)
+Foam::Istream& Foam::UIPstreamBase::read(char* data, std::streamsize count)
 {
     if (count)
     {
@@ -398,7 +461,7 @@ Foam::Istream& Foam::UIPstream::read(char* data, std::streamsize count)
 }
 
 
-Foam::Istream& Foam::UIPstream::readRaw(char* data, std::streamsize count)
+Foam::Istream& Foam::UIPstreamBase::readRaw(char* data, std::streamsize count)
 {
     // No check for format() == BINARY since this is either done in the
     // beginRawRead() method, or the caller knows what they are doing.
@@ -409,7 +472,7 @@ Foam::Istream& Foam::UIPstream::readRaw(char* data, std::streamsize count)
 }
 
 
-bool Foam::UIPstream::beginRawRead()
+bool Foam::UIPstreamBase::beginRawRead()
 {
     if (format() != BINARY)
     {
@@ -427,13 +490,30 @@ bool Foam::UIPstream::beginRawRead()
 }
 
 
-void Foam::UIPstream::rewind()
+// Not needed yet
+///
+/// //- The current get position (tellg) in the buffer
+/// label pos() const;
+///
+/// Foam::label Foam::UIPstreamBase::pos() const
+/// {
+///     return recvBufPos_;
+/// }
+
+
+void Foam::UIPstreamBase::rewind()
 {
-    recvBufPos_ = 0;
+    recvBufPos_ = 0;  // Assume the entire buffer is for us to read from
+    setOpened();
+    setGood();
+    if (recvBuf_.empty() || !messageSize_)
+    {
+        setEof();
+    }
 }
 
 
-void Foam::UIPstream::print(Ostream& os) const
+void Foam::UIPstreamBase::print(Ostream& os) const
 {
     os  << "Reading from processor " << fromProcNo_
         << " using communicator " << comm_

src/OpenFOAM/db/IOstreams/Pstreams/UOPstream.H

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2014 OpenFOAM Foundation
-    Copyright (C) 2017-2021 OpenCFD Ltd.
+    Copyright (C) 2017-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -25,21 +25,22 @@ License
     along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
 
 Class
-    Foam::UOPstream
+    Foam::UOPstreamBase
 
 Description
-    Output inter-processor communications stream operating on external
+    Base class for output inter-processor communications stream
-    buffer.
+    (ie, parallel streams).
+    Not to be used directly, thus contructors are protected.
 
 SourceFiles
-    UOPstream.C
+    UOPstreamBase.C
 
 \*---------------------------------------------------------------------------*/
 
 #include "Pstream.H"
 
-#ifndef UOPstream_H
+#ifndef Foam_UOPstream_H
-#define UOPstream_H
+#define Foam_UOPstream_H
 
 #include "UPstream.H"
 #include "Ostream.H"
@@ -52,27 +53,14 @@ namespace Foam
 {
 
 /*---------------------------------------------------------------------------*\
-                          Class UOPstream Declaration
+                        Class UOPstreamBase Declaration
 \*---------------------------------------------------------------------------*/
 
-class UOPstream
+class UOPstreamBase
 :
     public UPstream,
     public Ostream
 {
-    // Private Data
-
-        int toProcNo_;
-
-        DynamicList<char>& sendBuf_;
-
-        const int tag_;
-
-        const label comm_;
-
-        const bool sendAtDestruct_;
-
-
     // Private Member Functions
 
         //- Prepare send buffer for count bytes of output,
@@ -100,14 +88,27 @@ class UOPstream
         inline void putString(const std::string& str);
 
 
-public:
+protected:
 
-    // Constructors
+    // Protected Data
+
+        int toProcNo_;
+
+        DynamicList<char>& sendBuf_;
+
+        const int tag_;
+
+        const label comm_;
+
+        const bool sendAtDestruct_;
+
+
+    // Protected Constructors
 
         //- Construct given process index to write to using the given
         //- attached send buffer, optional communication characteristics
         //- and IO format
-        UOPstream
+        UOPstreamBase
         (
             const commsTypes commsType,
             const int toProcNo,
@@ -119,11 +120,13 @@ public:
         );
 
         //- Construct given buffers
-        UOPstream(const int toProcNo, PstreamBuffers& buffers);
+        UOPstreamBase(const int toProcNo, PstreamBuffers& buffers);
 
 
+public:
+
     //- Destructor.
-    ~UOPstream();
+    virtual ~UOPstreamBase();
 
 
     // Member Functions
@@ -139,17 +142,6 @@ public:
 
         // Write Functions
 
-            //- Write given buffer to given processor
-            static bool write
-            (
-                const commsTypes commsType,
-                const int toProcNo,
-                const char* buf,
-                const std::streamsize bufSize,
-                const int tag = UPstream::msgType(),
-                const label communicator = UPstream::worldComm
-            );
-
             //- Write token to stream or otherwise handle it.
             //  \return false if the token type was not handled by this method
             virtual bool write(const token& tok);
@@ -259,10 +251,16 @@ public:
             }
 
 
+    // Positioning
+
+        //- Rewind the send buffer for overwriting
+        virtual void rewind();
+
+
     // Edit
 
         //- Set flags of stream
-            ios_base::fmtflags flags(const ios_base::fmtflags)
+        virtual ios_base::fmtflags flags(const ios_base::fmtflags)
         {
             return ios_base::fmtflags(0);
         }
@@ -275,6 +273,133 @@ public:
 };
 
 
+/*---------------------------------------------------------------------------*\
+                          Class UOPstream Declaration
+\*---------------------------------------------------------------------------*/
+
+//- Output inter-processor communications stream
+//- using MPI send/recv etc. - operating on external buffer.
+class UOPstream
+:
+    public UOPstreamBase
+{
+    // Private Member Functions
+
+        //- Final buffer send, called by destructor
+        bool bufferIPCsend();
+
+
+public:
+
+    // Constructors
+
+        //- Construct given process index to write to using the given
+        //- attached send buffer, optional communication characteristics
+        //- and IO format
+        UOPstream
+        (
+            const commsTypes commsType,
+            const int toProcNo,
+            DynamicList<char>& sendBuf,
+            const int tag = UPstream::msgType(),
+            const label comm = UPstream::worldComm,
+            const bool sendAtDestruct = true,
+            IOstreamOption::streamFormat fmt = IOstreamOption::BINARY
+        );
+
+        //- Construct given buffers
+        UOPstream(const int toProcNo, PstreamBuffers& buffers);
+
+
+    //- Destructor, usually sends buffer on destruct.
+    virtual ~UOPstream();
+
+
+    // Member Functions
+
+        //- Use all write methods from base class
+        using UOPstreamBase::write;
+
+
+    // Static Functions
+
+        //- Write buffer contents to given processor
+        //  \return True on success
+        static bool write
+        (
+            const commsTypes commsType,
+            const int toProcNo,
+            const char* buf,
+            const std::streamsize bufSize,
+            const int tag = UPstream::msgType(),
+            const label comm = UPstream::worldComm
+        );
+};
+
+
+/*---------------------------------------------------------------------------*\
+                         Class UOPBstream Declaration
+\*---------------------------------------------------------------------------*/
+
+//- Output inter-processor communications stream
+//- using MPI broadcast - operating on external buffer.
+//
+// \note does not use commsType, tag etc.
+class UOPBstream
+:
+    public UOPstreamBase
+{
+    // Private Member Functions
+
+        //- Final buffer send, called by destructor
+        bool bufferIPCsend();
+
+
+public:
+
+    // Constructors
+
+        //- Construct given process index to write to using the given
+        //- attached send buffer, optional communication characteristics
+        //- and IO format
+        UOPBstream
+        (
+            const commsTypes commsType,     //!< ignored
+            const int toProcNo,             //!< normally UPstream::masterNo()
+            DynamicList<char>& sendBuf,
+            const int tag = UPstream::msgType(),  //!< ignored
+            const label comm = UPstream::worldComm,
+            const bool sendAtDestruct = true,
+            IOstreamOption::streamFormat fmt = IOstreamOption::BINARY
+        );
+
+
+    //- Destructor, usually sends buffer on destruct.
+    virtual ~UOPBstream();
+
+
+    // Member Functions
+
+        //- Use all write methods from base
+        using UOPstreamBase::write;
+
+
+    // Static Functions
+
+        //- Wrapped version of UPstream::broadcast with const-cast
+        //  \return True on success
+        static bool write
+        (
+            const commsTypes commsType,     //!< ignored
+            const int rootProcNo,           //!< normally UPstream::masterNo()
+            const char* buf,
+            const std::streamsize bufSize,
+            const int tag = UPstream::msgType(),  //!< ignored
+            const label comm = UPstream::worldComm
+        );
+};
+
+
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
 } // End namespace Foam

src/OpenFOAM/db/IOstreams/Pstreams/UOPstreamBase.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2017 OpenFOAM Foundation
-    Copyright (C) 2016-2021 OpenCFD Ltd.
+    Copyright (C) 2016-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -52,7 +52,7 @@ inline static label byteAlign(const label pos, const size_t align)
 
 // * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
 
-inline void Foam::UOPstream::prepareBuffer
+inline void Foam::UOPstreamBase::prepareBuffer
 (
     const size_t count,
     const size_t align
@@ -75,13 +75,13 @@ inline void Foam::UOPstream::prepareBuffer
 
 
 template<class T>
-inline void Foam::UOPstream::writeToBuffer(const T& val)
+inline void Foam::UOPstreamBase::writeToBuffer(const T& val)
 {
     writeToBuffer(&val, sizeof(T), sizeof(T));
 }
 
 
-inline void Foam::UOPstream::writeToBuffer
+inline void Foam::UOPstreamBase::writeToBuffer
 (
     const void* data,
     const size_t count,
@@ -111,7 +111,7 @@ inline void Foam::UOPstream::writeToBuffer
 }
 
 
-inline void Foam::UOPstream::putChar(const char c)
+inline void Foam::UOPstreamBase::putChar(const char c)
 {
     if (!sendBuf_.capacity())
     {
@@ -121,7 +121,7 @@ inline void Foam::UOPstream::putChar(const char c)
 }
 
 
-inline void Foam::UOPstream::putString(const std::string& str)
+inline void Foam::UOPstreamBase::putString(const std::string& str)
 {
     const size_t len = str.size();
     writeToBuffer(len);
@@ -131,7 +131,7 @@ inline void Foam::UOPstream::putString(const std::string& str)
 
 // * * * * * * * * * * * * * * * * Constructor * * * * * * * * * * * * * * * //
 
-Foam::UOPstream::UOPstream
+Foam::UOPstreamBase::UOPstreamBase
 (
     const commsTypes commsType,
     const int toProcNo,
@@ -155,15 +155,15 @@ Foam::UOPstream::UOPstream
 }
 
 
-Foam::UOPstream::UOPstream(const int toProcNo, PstreamBuffers& buffers)
+Foam::UOPstreamBase::UOPstreamBase(const int toProcNo, PstreamBuffers& buffers)
 :
-    UPstream(buffers.commsType_),
+    UPstream(buffers.commsType()),
-    Ostream(buffers.format_, IOstreamOption::currentVersion),
+    Ostream(buffers.format(), IOstreamOption::currentVersion),
     toProcNo_(toProcNo),
     sendBuf_(buffers.sendBuf_[toProcNo]),
-    tag_(buffers.tag_),
+    tag_(buffers.tag()),
-    comm_(buffers.comm_),
+    comm_(buffers.comm()),
-    sendAtDestruct_(buffers.commsType_ != UPstream::commsTypes::nonBlocking)
+    sendAtDestruct_(buffers.commsType() != UPstream::commsTypes::nonBlocking)
 {
     setOpened();
     setGood();
@@ -172,35 +172,13 @@ Foam::UOPstream::UOPstream(const int toProcNo, PstreamBuffers& buffers)
 
 // * * * * * * * * * * * * * * * * Destructor  * * * * * * * * * * * * * * * //
 
-Foam::UOPstream::~UOPstream()
+Foam::UOPstreamBase::~UOPstreamBase()
-{
+{}
-    if (sendAtDestruct_)
-    {
-        if
-        (
-            !UOPstream::write
-            (
-                commsType_,
-                toProcNo_,
-                sendBuf_.cdata(),
-                sendBuf_.size(),
-                tag_,
-                comm_
-            )
-        )
-        {
-            FatalErrorInFunction
-                << "Failed sending outgoing message of size " << sendBuf_.size()
-                << " to processor " << toProcNo_
-                << Foam::abort(FatalError);
-        }
-    }
-}
 
 
 // * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * * //
 
-bool Foam::UOPstream::write(const token& tok)
+bool Foam::UOPstreamBase::write(const token& tok)
 {
     // Direct token handling only for some types
 
@@ -244,7 +222,7 @@ bool Foam::UOPstream::write(const token& tok)
 }
 
 
-Foam::Ostream& Foam::UOPstream::write(const char c)
+Foam::Ostream& Foam::UOPstreamBase::write(const char c)
 {
     if (!isspace(c))
     {
@@ -255,7 +233,7 @@ Foam::Ostream& Foam::UOPstream::write(const char c)
 }
 
 
-Foam::Ostream& Foam::UOPstream::write(const char* str)
+Foam::Ostream& Foam::UOPstreamBase::write(const char* str)
 {
     const word nonWhiteChars(string::validate<word>(str));
 
@@ -272,7 +250,7 @@ Foam::Ostream& Foam::UOPstream::write(const char* str)
 }
 
 
-Foam::Ostream& Foam::UOPstream::write(const word& str)
+Foam::Ostream& Foam::UOPstreamBase::write(const word& str)
 {
     putChar(token::tokenType::WORD);
     putString(str);
@@ -281,7 +259,7 @@ Foam::Ostream& Foam::UOPstream::write(const word& str)
 }
 
 
-Foam::Ostream& Foam::UOPstream::write(const string& str)
+Foam::Ostream& Foam::UOPstreamBase::write(const string& str)
 {
     putChar(token::tokenType::STRING);
     putString(str);
@@ -290,7 +268,7 @@ Foam::Ostream& Foam::UOPstream::write(const string& str)
 }
 
 
-Foam::Ostream& Foam::UOPstream::writeQuoted
+Foam::Ostream& Foam::UOPstreamBase::writeQuoted
 (
     const std::string& str,
     const bool quoted
@@ -310,7 +288,7 @@ Foam::Ostream& Foam::UOPstream::writeQuoted
 }
 
 
-Foam::Ostream& Foam::UOPstream::write(const int32_t val)
+Foam::Ostream& Foam::UOPstreamBase::write(const int32_t val)
 {
     putChar(token::tokenType::LABEL);
     writeToBuffer(val);
@@ -318,7 +296,7 @@ Foam::Ostream& Foam::UOPstream::write(const int32_t val)
 }
 
 
-Foam::Ostream& Foam::UOPstream::write(const int64_t val)
+Foam::Ostream& Foam::UOPstreamBase::write(const int64_t val)
 {
     putChar(token::tokenType::LABEL);
     writeToBuffer(val);
@@ -326,7 +304,7 @@ Foam::Ostream& Foam::UOPstream::write(const int64_t val)
 }
 
 
-Foam::Ostream& Foam::UOPstream::write(const floatScalar val)
+Foam::Ostream& Foam::UOPstreamBase::write(const floatScalar val)
 {
     putChar(token::tokenType::FLOAT);
     writeToBuffer(val);
@@ -334,7 +312,7 @@ Foam::Ostream& Foam::UOPstream::write(const floatScalar val)
 }
 
 
-Foam::Ostream& Foam::UOPstream::write(const doubleScalar val)
+Foam::Ostream& Foam::UOPstreamBase::write(const doubleScalar val)
 {
     putChar(token::tokenType::DOUBLE);
     writeToBuffer(val);
@@ -342,7 +320,7 @@ Foam::Ostream& Foam::UOPstream::write(const doubleScalar val)
 }
 
 
-Foam::Ostream& Foam::UOPstream::write(const char* data, std::streamsize count)
+Foam::Ostream& Foam::UOPstreamBase::write(const char* data, std::streamsize count)
 {
     if (format() != BINARY)
     {
@@ -358,7 +336,7 @@ Foam::Ostream& Foam::UOPstream::write(const char* data, std::streamsize count)
 }
 
 
-Foam::Ostream& Foam::UOPstream::writeRaw
+Foam::Ostream& Foam::UOPstreamBase::writeRaw
 (
     const char* data,
     std::streamsize count
@@ -374,7 +352,7 @@ Foam::Ostream& Foam::UOPstream::writeRaw
 }
 
 
-bool Foam::UOPstream::beginRawWrite(std::streamsize count)
+bool Foam::UOPstreamBase::beginRawWrite(std::streamsize count)
 {
     if (format() != BINARY)
     {
@@ -391,7 +369,26 @@ bool Foam::UOPstream::beginRawWrite(std::streamsize count)
 }
 
 
-void Foam::UOPstream::print(Ostream& os) const
+// Not needed yet
+///
+/// //- The current put position (tellp) in the buffer
+/// label pos() const;
+///
+/// Foam::label Foam::UOPstreamBase::pos() const
+/// {
+///     return sendBuf_.size();
+/// }
+
+
+void Foam::UOPstreamBase::rewind()
+{
+    sendBuf_.clear();  // Overwrite into buffer
+    setOpened();
+    setGood();
+}
+
+
+void Foam::UOPstreamBase::print(Ostream& os) const
 {
     os  << "Writing from processor " << toProcNo_
         << " to processor " << myProcNo() << " in communicator " << comm_

src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2017 OpenFOAM Foundation
-    Copyright (C) 2015-2021 OpenCFD Ltd.
+    Copyright (C) 2015-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -51,6 +51,43 @@ Foam::UPstream::commsTypeNames
 });
 
 
+// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * //
+
+void Foam::UPstream::broadcast
+(
+    std::string& str,
+    const label comm,
+    const int rootProcNo
+)
+{
+    if (UPstream::parRun() && UPstream::nProcs(comm) > 1)
+    {
+        // Broadcast the string length
+        std::size_t len(str.length());
+
+        UPstream::broadcast
+        (
+            reinterpret_cast<char*>(&len),
+            sizeof(std::size_t),
+            comm,
+            rootProcNo
+        );
+
+        if (!UPstream::master(comm))
+        {
+            // Do not touch string on the master even although it would
+            // be a no-op. We are truly paranoid.
+            str.resize(len);
+        }
+
+        if (len)
+        {
+            UPstream::broadcast(&str[0], len, comm, rootProcNo);
+        }
+    }
+}
+
+
 // * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
 
 void Foam::UPstream::setParRun(const label nProcs, const bool haveThreads)
@@ -414,7 +451,7 @@ registerOptSwitch
 
 int Foam::UPstream::nProcsSimpleSum
 (
-    Foam::debug::optimisationSwitch("nProcsSimpleSum", 16)
+    Foam::debug::optimisationSwitch("nProcsSimpleSum", 0)
 );
 registerOptSwitch
 (

src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H

@@ -34,9 +34,6 @@ SourceFiles
     UPstream.C
     UPstreamCommsStruct.C
     UPstreamTemplates.C
-    combineGatherScatter.C
-    gatherScatter.C
-    gatherScatterList.C
 
 \*---------------------------------------------------------------------------*/
 
@@ -69,9 +66,9 @@ public:
     //- Types of communications
     enum class commsTypes : char
     {
-        blocking,
+        blocking,       //!< "blocking"
-        scheduled,
+        scheduled,      //!< "scheduled"
-        nonBlocking
+        nonBlocking     //!< "nonBlocking"
     };
 
     //- Names of the communication types
@@ -104,7 +101,7 @@ public:
             // Constructors
 
                 //- Default construct. Above == -1
-                commsStruct();
+                commsStruct() noexcept;
 
                 //- Construct from components
                 commsStruct
@@ -180,6 +177,12 @@ public:
 
 private:
 
+    // Private Data
+
+        //- Communications type of this stream
+        commsTypes commsType_;
+
+
     // Private Static Data
 
         //- By default this is not a parallel run
@@ -253,14 +256,6 @@ private:
         );
 
 
-protected:
-
-    // Protected Data
-
-        //- Communications type of this stream
-        commsTypes commsType_;
-
-
 public:
 
     // Declare name of the class and its debug switch
@@ -274,8 +269,7 @@ public:
         //- in accuracy
         static bool floatTransfer;
 
-        //- Number of processors at which the sum algorithm changes from linear
+        //- Number of processors to change from linear to tree communication
-        //- to tree
         static int nProcsSimpleSum;
 
         //- Default commsType
@@ -413,11 +407,11 @@ public:
 
             static int allocateTag(const char*);
 
-            static int allocateTag(const word&);
+            static int allocateTag(const std::string&);
 
             static void freeTag(const char*, const int tag);
 
-            static void freeTag(const word&, const int tag);
+            static void freeTag(const std::string&, const int tag);
 
 
         //- Set as parallel run on/off.
@@ -537,13 +531,28 @@ public:
             return treeCommunication_[communicator];
         }
 
+        //- Communication schedule for linear/tree all-to-master (proc 0).
+        //- Chooses based on the value of UPstream::nProcsSimpleSum
+        static const List<commsStruct>& whichCommunication
+        (
+            const label communicator = worldComm
+        )
+        {
+            return
+            (
+                nProcs(communicator) < nProcsSimpleSum
+              ? linearCommunication_[communicator]
+              : treeCommunication_[communicator]
+            );
+        }
+
+
         //- Message tag of standard messages
         static int& msgType() noexcept
         {
             return msgType_;
         }
 
-
         //- Get the communications type of the stream
         commsTypes commsType() const noexcept
         {
@@ -551,6 +560,7 @@ public:
         }
 
         //- Set the communications type of the stream
+        //  \return the previous value
         commsTypes commsType(const commsTypes ct) noexcept
         {
             commsTypes old(commsType_);
@@ -668,6 +678,28 @@ public:
         );
 
 
+    // Broadcast Functions
+
+        //- Broadcast buffer contents to all processes in communicator.
+        //- The sizes must match on all processes.
+        //  \return True on success
+        static bool broadcast
+        (
+            char* buf,
+            const std::streamsize bufSize,
+            const label communicator = worldComm,
+            const int rootProcNo = masterNo()
+        );
+
+        //- Broadcast string content to all processes in communicator.
+        static void broadcast
+        (
+            std::string& str,
+            const label communicator = worldComm,
+            const int rootProcNo = masterNo()
+        );
+
+
     // Housekeeping
 
         //- Process index of first sub-process
@@ -691,11 +723,11 @@ Ostream& operator<<(Ostream&, const UPstream::commsStruct&);
 // Template specialisation for access of commsStruct
 template<>
 UPstream::commsStruct&
-UList<UPstream::commsStruct>::operator[](const label);
+UList<UPstream::commsStruct>::operator[](const label procID);
 
 template<>
 const UPstream::commsStruct&
-UList<UPstream::commsStruct>::operator[](const label) const;
+UList<UPstream::commsStruct>::operator[](const label procID) const;
 
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //

src/OpenFOAM/db/IOstreams/Pstreams/UPstreamCommsStruct.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2016 OpenFOAM Foundation
-    Copyright (C) 2021 OpenCFD Ltd.
+    Copyright (C) 2021-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -30,7 +30,7 @@ License
 
 // * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
 
-Foam::UPstream::commsStruct::commsStruct()
+Foam::UPstream::commsStruct::commsStruct() noexcept
 :
     above_(-1),
     below_(),

src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2017-2018 OpenFOAM Foundation
-    Copyright (C) 2019-2021 OpenCFD Ltd.
+    Copyright (C) 2019-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -2072,24 +2072,11 @@ bool Foam::fileOperations::masterUncollatedFileOperation::read
         Pstream::scatter(io.note());    //, Pstream::msgType(), comm_);
 
 
-        // scatter operation for regIOobjects
+        // Scatter operation for regIOobjects
 
-        // Get my communication order
+        // My communication order
-        // const List<Pstream::commsStruct>& comms =
+        const auto& comms = Pstream::whichCommunication(Pstream::worldComm);
-        //(
+        const auto& myComm = comms[Pstream::myProcNo(Pstream::worldComm)];
-        //    (Pstream::nProcs(comm_) < Pstream::nProcsSimpleSum)
-        //  ? Pstream::linearCommunication(comm_)
-        //  : Pstream::treeCommunication(comm_)
-        //);
-        // const Pstream::commsStruct& myComm = comms[Pstream::myProcNo(comm_)];
-        const List<Pstream::commsStruct>& comms =
-        (
-            (Pstream::nProcs(Pstream::worldComm) < Pstream::nProcsSimpleSum)
-          ? Pstream::linearCommunication(Pstream::worldComm)
-          : Pstream::treeCommunication(Pstream::worldComm)
-        );
-        const Pstream::commsStruct& myComm =
-            comms[Pstream::myProcNo(Pstream::worldComm)];
 
         // Receive from up
         if (myComm.above() != -1)

src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2017 OpenFOAM Foundation
-    Copyright (C) 2020-2021 OpenCFD Ltd.
+    Copyright (C) 2020-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -654,14 +654,9 @@ bool Foam::fileOperations::uncollatedFileOperation::read
         Pstream::scatter(io.headerClassName());
         Pstream::scatter(io.note());
 
-        // Get my communication order
+        // My communication order
-        const List<Pstream::commsStruct>& comms =
+        const auto& comms = Pstream::whichCommunication();
-        (
+        const auto& myComm = comms[Pstream::myProcNo()];
-            (Pstream::nProcs() < Pstream::nProcsSimpleSum)
-          ? Pstream::linearCommunication()
-          : Pstream::treeCommunication()
-        );
-        const Pstream::commsStruct& myComm = comms[Pstream::myProcNo()];
 
         // Receive from up
         if (myComm.above() != -1)

src/OpenFOAM/global/profiling/profilingPstream.H

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2019-2020 OpenCFD Ltd.
+    Copyright (C) 2019-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -35,8 +35,8 @@ SourceFiles
 
 \*---------------------------------------------------------------------------*/
 
-#ifndef profilingPstream_H
+#ifndef Foam_profilingPstream_H
-#define profilingPstream_H
+#define Foam_profilingPstream_H
 
 #include "cpuTime.H"
 #include "FixedList.H"
@@ -62,13 +62,14 @@ public:
         {
             GATHER = 0,
             SCATTER,
+            BROADCAST,
             REDUCE,
             WAIT,
             ALL_TO_ALL
         };
 
         //- The timing values
-        typedef FixedList<double, 5> timingList;
+        typedef FixedList<double, 6> timingList;
 
 
 private:
@@ -165,6 +166,12 @@ public:
             addTime(SCATTER);
         }
 
+        //- Add time increment to broadcastTime
+        inline static void addBroadcastTime()
+        {
+            addTime(BROADCAST);
+        }
+
         //- Add time increment to reduceTime
         inline static void addReduceTime()
         {

src/OpenFOAM/matrices/lduMatrix/lduAddressing/lduInterface/processorLduInterface.C

@@ -38,11 +38,12 @@ namespace Foam
 
 // * * * * * * * * * * * * * Private Member Functions *  * * * * * * * * * * //
 
-void Foam::processorLduInterface::resizeBuf(List<char>& buf, const label size)
+void Foam::processorLduInterface::resizeBuf(List<char>& buf, const label len)
 {
-    if (buf.size() < size)
+    if (buf.size() < len)
     {
-        buf.resize(size);
+        // Use nocopy variant since it will be overwritten
+        buf.resize_nocopy(len);
     }
 }
 

src/OpenFOAM/matrices/lduMatrix/lduAddressing/lduInterface/processorLduInterface.H

@@ -36,8 +36,8 @@ SourceFiles
 
 \*---------------------------------------------------------------------------*/
 
-#ifndef processorLduInterface_H
+#ifndef Foam_processorLduInterface_H
-#define processorLduInterface_H
+#define Foam_processorLduInterface_H
 
 #include "lduInterface.H"
 #include "primitiveFieldsFwd.H"
@@ -67,7 +67,7 @@ class processorLduInterface
     // Private Member Functions
 
         //- Increase buffer size if required
-        static void resizeBuf(List<char>& buf, const label size);
+        static void resizeBuf(List<char>& buf, const label len);
 
 
 public:
@@ -78,7 +78,7 @@ public:
 
     // Constructors
 
-        //- Construct null
+        //- Default construct
         processorLduInterface() = default;
 
 

src/OpenFOAM/matrices/lduMatrix/lduAddressing/lduInterface/processorLduInterfaceTemplates.C

@@ -39,7 +39,7 @@ void Foam::processorLduInterface::send
     const UList<Type>& f
 ) const
 {
-    label nBytes = f.byteSize();
+    const label nBytes = f.byteSize();
 
     if
     (

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverScale.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2017 OpenFOAM Foundation
-    Copyright (C) 2017-2019 OpenCFD Ltd.
+    Copyright (C) 2017-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -27,7 +27,7 @@ License
 \*---------------------------------------------------------------------------*/
 
 #include "GAMGSolver.H"
-#include "vector2D.H"
+#include "FixedList.H"
 
 // * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
 
@@ -58,21 +58,21 @@ void Foam::GAMGSolver::scale
     const solveScalar* const __restrict__ AcfPtr = Acf.begin();
 
 
-    solveScalar scalingFactorNum = 0.0;
+    FixedList<solveScalar, 2> scalingFactor(Zero);
-    solveScalar scalingFactorDenom = 0.0;
 
     for (label i=0; i<nCells; i++)
     {
-        scalingFactorNum += sourcePtr[i]*fieldPtr[i];
+        scalingFactor[0] += fieldPtr[i]*sourcePtr[i];
-        scalingFactorDenom += AcfPtr[i]*fieldPtr[i];
+        scalingFactor[1] += fieldPtr[i]*AcfPtr[i];
     }
 
-    Vector2D<solveScalar> scalingVector(scalingFactorNum, scalingFactorDenom);
+    A.mesh().reduce(scalingFactor, sumOp<solveScalar>());
-    A.mesh().reduce(scalingVector, sumOp<Vector2D<solveScalar>>());
 
     const solveScalar sf =
-        scalingVector.x()
+    (
-       /stabilise(scalingVector.y(), pTraits<solveScalar>::vsmall);
+        scalingFactor[0]
+      / stabilise(scalingFactor[1], pTraits<solveScalar>::vsmall)
+    );
 
     if (debug >= 2)
     {

src/OpenFOAM/meshes/ProcessorTopology/ProcessorTopology.C

@@ -29,7 +29,6 @@ License
 #include "ListOps.H"
 #include "Pstream.H"
 #include "commSchedule.H"
-#include "boolList.H"
 
 // * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
 
@@ -50,22 +49,16 @@ Foam::labelList Foam::ProcessorTopology<Container, ProcPatch>::procNeighbours
 
     forAll(patches, patchi)
     {
-        const typename Container::const_reference patch = patches[patchi];
+        const auto* cpp = isA<ProcPatch>(patches[patchi]);
-
+        if (cpp)
-        if (isA<ProcPatch>(patch))
         {
-            const ProcPatch& procPatch =
+            const label nbrProci = cpp->neighbProcNo();
-                refCast<const ProcPatch>(patch);
 
-            label pNeighbProcNo = procPatch.neighbProcNo();
+            if (!isNeighbourProc[nbrProci])
-
-            if (!isNeighbourProc[pNeighbProcNo])
             {
-                nNeighbours++;
+                isNeighbourProc[nbrProci] = true;
-
+                maxNb = max(maxNb, nbrProci);
-                maxNb = max(maxNb, procPatch.neighbProcNo());
+                ++nNeighbours;
-
-                isNeighbourProc[pNeighbProcNo] = true;
             }
         }
     }
@@ -87,15 +80,13 @@ Foam::labelList Foam::ProcessorTopology<Container, ProcPatch>::procNeighbours
 
     forAll(patches, patchi)
     {
-        const typename Container::const_reference patch = patches[patchi];
+        const auto* cpp = isA<ProcPatch>(patches[patchi]);
-
+        if (cpp)
-        if (isA<ProcPatch>(patch))
         {
-            const ProcPatch& procPatch =
+            const label nbrProci = cpp->neighbProcNo();
-                refCast<const ProcPatch>(patch);
 
-            // Construct reverse map
+            // Reverse map
-            procPatchMap_[procPatch.neighbProcNo()] = patchi;
+            procPatchMap_[nbrProci] = patchi;
         }
     }
 
@@ -113,7 +104,7 @@ Foam::ProcessorTopology<Container, ProcPatch>::ProcessorTopology
 )
 :
     labelListList(Pstream::nProcs(comm)),
-    patchSchedule_(2*patches.size())
+    patchSchedule_()
 {
     if (Pstream::parRun())
     {
@@ -132,6 +123,8 @@ Foam::ProcessorTopology<Container, ProcPatch>::ProcessorTopology
      && Pstream::defaultCommsType == Pstream::commsTypes::scheduled
     )
     {
+        patchSchedule_.resize(2*patches.size());
+
         label patchEvali = 0;
 
         // 1. All non-processor patches
@@ -155,21 +148,21 @@ Foam::ProcessorTopology<Container, ProcPatch>::ProcessorTopology
         // to determine the schedule. Each processor pair stands for both
         // send and receive.
         label nComms = 0;
-        forAll(*this, proci)
+        for (const labelList& neighbours : *this)
         {
-            nComms += operator[](proci).size();
+            nComms += neighbours.size();
         }
         DynamicList<labelPair> comms(nComms);
 
         forAll(*this, proci)
         {
-            const labelList& nbrs = operator[](proci);
+            const labelList& neighbours = operator[](proci);
 
-            forAll(nbrs, i)
+            for (const label nbrProci : neighbours)
             {
-                if (proci < nbrs[i])
+                if (proci < nbrProci)
                 {
-                    comms.append(labelPair(proci, nbrs[i]));
+                    comms.append(labelPair(proci, nbrProci));
                 }
             }
         }
@@ -185,10 +178,8 @@ Foam::ProcessorTopology<Container, ProcPatch>::ProcessorTopology
             ).procSchedule()[Pstream::myProcNo(comm)]
         );
 
-        forAll(mySchedule, iter)
+        for (const label commI : mySchedule)
         {
-            label commI = mySchedule[iter];
-
             // Get the other processor
             label nb = comms[commI][0];
             if (nb == Pstream::myProcNo(comm))

src/OpenFOAM/meshes/ProcessorTopology/ProcessorTopology.H

@@ -40,8 +40,8 @@ SourceFiles
 
 \*---------------------------------------------------------------------------*/
 
-#ifndef ProcessorTopology_H
+#ifndef Foam_ProcessorTopology_H
-#define ProcessorTopology_H
+#define Foam_ProcessorTopology_H
 
 #include "labelList.H"
 #include "lduSchedule.H"
@@ -60,10 +60,7 @@ class ProcessorTopology
 :
     public labelListList
 {
-
+    // Private Data
-private:
-
-    // Private data
 
         //- Local map from neighbour proc to patchi. Different per processor!
         //  -1 or patchi for connection to procID
@@ -76,8 +73,8 @@ private:
 
     // Private Member Functions
 
-        //- Return all neighbouring processors of this processor. Set
+        //- Return all neighbouring processors of this processor.
-        //  procPatchMap_.
+        //  Sets procPatchMap_.
         labelList procNeighbours(const label nProcs, const Container&);
 
 public:
@@ -88,25 +85,27 @@ public:
         ProcessorTopology(const Container& patches, const label comm);
 
 
+    // Static Functions
+
+        //- Calculate non-blocking (i.e. unscheduled) schedule
+        static lduSchedule nonBlockingSchedule(const Container& patches);
+
+
     // Member Functions
 
-        //- From neighbour processor to index in boundaryMesh. Local information
+        //- From neighbour processor to index in boundaryMesh.
-        //  (so not same over all processors)
+        //  Local information (so not same over all processors)
-        const labelList& procPatchMap() const
+        const labelList& procPatchMap() const noexcept
         {
             return procPatchMap_;
         }
 
         //- Order in which the patches should be initialised/evaluated
-        //  corresponding to the schedule
+        //- corresponding to the schedule
-        const lduSchedule& patchSchedule() const
+        const lduSchedule& patchSchedule() const noexcept
         {
             return patchSchedule_;
         }
-
-        //- Calculate non-blocking (i.e. unscheduled) schedule
-        static lduSchedule nonBlockingSchedule(const Container& patches);
-
 };
 
 

src/OpenFOAM/meshes/polyMesh/globalMeshData/globalMeshData.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2017 OpenFOAM Foundation
-    Copyright (C) 2015-2021 OpenCFD Ltd.
+    Copyright (C) 2015-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -388,20 +388,20 @@ void Foam::globalMeshData::calcSharedEdges() const
         }
         countSharedEdges(localShared, globalShared, sharedEdgeI);
 
-        // Receive data from slaves and insert
+        // Receive data and insert
         if (Pstream::parRun())
         {
-            for (const int slave : Pstream::subProcs())
+            for (const int proci : Pstream::subProcs())
             {
                 // Receive the edges using shared points from the slave.
-                IPstream fromSlave(Pstream::commsTypes::blocking, slave);
+                IPstream fromProc(Pstream::commsTypes::blocking, proci);
-                EdgeMap<labelList> procSharedEdges(fromSlave);
+                EdgeMap<labelList> procSharedEdges(fromProc);
 
                 if (debug)
                 {
                     Pout<< "globalMeshData::calcSharedEdges : "
                         << "Merging in from proc"
-                        << Foam::name(slave) << " : " << procSharedEdges.size()
+                        << proci << " : " << procSharedEdges.size()
                         << endl;
                 }
                 countSharedEdges(procSharedEdges, globalShared, sharedEdgeI);
@@ -430,23 +430,12 @@ void Foam::globalMeshData::calcSharedEdges() const
                     << " down to " << globalShared.size() << endl;
             }
         }
-
-
-        // Send back to slaves.
-        if (Pstream::parRun())
-        {
-            for (const int slave : Pstream::subProcs())
-            {
-                // Receive the edges using shared points from the slave.
-                OPstream toSlave(Pstream::commsTypes::blocking, slave);
-                toSlave << globalShared;
-            }
-        }
     }
     else
     {
-        // Send local edges to master
+        if (Pstream::parRun())
         {
+            // Send local edges to master
             OPstream toMaster
             (
                 Pstream::commsTypes::blocking,
@@ -454,17 +443,12 @@ void Foam::globalMeshData::calcSharedEdges() const
             );
             toMaster << localShared;
         }
-        // Receive merged edges from master.
-        {
-            IPstream fromMaster
-            (
-                Pstream::commsTypes::blocking,
-                Pstream::masterNo()
-            );
-            fromMaster >> globalShared;
-        }
     }
 
+    // Broadcast: merged edges to all
+    Pstream::broadcast(globalShared);  // == worldComm;
+
+
     // Now use the global shared edges list (globalShared) to classify my local
     // ones (localShared)
 
@@ -1896,14 +1880,14 @@ Foam::pointField Foam::globalMeshData::sharedPoints() const
             sharedPoints[sharedPointi] = mesh_.points()[pointLabels[i]];
         }
 
-        // Receive data from slaves and insert
+        // Receive data and insert
-        for (const int slave : Pstream::subProcs())
+        for (const int proci : Pstream::subProcs())
         {
-            IPstream fromSlave(Pstream::commsTypes::blocking, slave);
+            IPstream fromProc(Pstream::commsTypes::blocking, proci);
 
             labelList nbrSharedPointAddr;
             pointField nbrSharedPoints;
-            fromSlave >> nbrSharedPointAddr >> nbrSharedPoints;
+            fromProc >> nbrSharedPointAddr >> nbrSharedPoints;
 
             forAll(nbrSharedPointAddr, i)
             {
@@ -1912,24 +1896,12 @@ Foam::pointField Foam::globalMeshData::sharedPoints() const
                 sharedPoints[sharedPointi] = nbrSharedPoints[i];
             }
         }
-
-        // Send back
-        for (const int slave : Pstream::subProcs())
-        {
-            OPstream toSlave
-            (
-                Pstream::commsTypes::blocking,
-                slave,
-                sharedPoints.size_bytes()
-            );
-            toSlave << sharedPoints;
-        }
     }
     else
     {
-        // Slave:
+        if (Pstream::parRun())
-        // send points
         {
+            // Send address and points
             OPstream toMaster
             (
                 Pstream::commsTypes::blocking,
@@ -1937,19 +1909,13 @@ Foam::pointField Foam::globalMeshData::sharedPoints() const
             );
             toMaster
                 << pointAddr
-                << UIndirectList<point>(mesh_.points(), pointLabels)();
+                << pointField(mesh_.points(), pointLabels);
+        }
     }
 
-        // Receive sharedPoints
+    // Broadcast: sharedPoints to all
-        {
+    Pstream::broadcast(sharedPoints);  // == worldComm
-            IPstream fromMaster
+
-            (
-                Pstream::commsTypes::blocking,
-                Pstream::masterNo()
-            );
-            fromMaster >> sharedPoints;
-        }
-    }
 
     return sharedPoints;
 }

src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeBase.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2015-2017 OpenFOAM Foundation
-    Copyright (C) 2015-2021 OpenCFD Ltd.
+    Copyright (C) 2015-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -80,48 +80,31 @@ Foam::List<Foam::labelPair> Foam::mapDistributeBase::schedule
     }
 
 
-    // Reduce
+    // Gather/reduce
     if (Pstream::master(comm))
     {
         // Receive and merge
-        for (const int slave : Pstream::subProcs(comm))
+        for (const int proci : Pstream::subProcs(comm))
         {
-            IPstream fromSlave
+            IPstream fromProc
             (
                 Pstream::commsTypes::scheduled,
-                slave,
+                proci,
                 0,
                 tag,
                 comm
             );
-            List<labelPair> nbrData(fromSlave);
+            List<labelPair> nbrData(fromProc);
 
-            forAll(nbrData, i)
+            for (const labelPair& connection : nbrData)
             {
-                if (!allComms.found(nbrData[i]))
+                allComms.appendUniq(connection);
-                {
-                    label sz = allComms.size();
-                    allComms.setSize(sz+1);
-                    allComms[sz] = nbrData[i];
             }
         }
     }
-        // Send back
-        for (const int slave : Pstream::subProcs(comm))
-        {
-            OPstream toSlave
-            (
-                Pstream::commsTypes::scheduled,
-                slave,
-                0,
-                tag,
-                comm
-            );
-            toSlave << allComms;
-        }
-    }
     else
     {
+        if (Pstream::parRun())
         {
             OPstream toMaster
             (
@@ -133,19 +116,11 @@ Foam::List<Foam::labelPair> Foam::mapDistributeBase::schedule
             );
             toMaster << allComms;
         }
-        {
-            IPstream fromMaster
-            (
-                Pstream::commsTypes::scheduled,
-                Pstream::masterNo(),
-                0,
-                tag,
-                comm
-            );
-            fromMaster >> allComms;
-        }
     }
 
+    // Broadcast: send comms information to all
+    Pstream::broadcast(allComms, comm);
+
 
     // Determine my schedule.
     labelList mySchedule
@@ -158,7 +133,7 @@ Foam::List<Foam::labelPair> Foam::mapDistributeBase::schedule
     );
 
     // Processors involved in my schedule
-    return List<labelPair>(UIndirectList<labelPair>(allComms, mySchedule));
+    return List<labelPair>(allComms, mySchedule);
 
 
     //if (debug)

src/OpenFOAM/meshes/polyMesh/syncTools/syncToolsTemplates.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2017 OpenFOAM Foundation
-    Copyright (C) 2015-2021 OpenCFD Ltd.
+    Copyright (C) 2015-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -127,6 +127,7 @@ void Foam::syncTools::syncPointMap
 
     if (Pstream::parRun())
     {
+        DynamicList<label> neighbProcs;
         PstreamBuffers pBufs(Pstream::commsTypes::nonBlocking);
 
         // Send
@@ -137,6 +138,7 @@ void Foam::syncTools::syncPointMap
             if (ppp && pp.nPoints())
             {
                 const auto& procPatch = *ppp;
+                const label nbrProci = procPatch.neighbProcNo();
 
                 // Get data per patchPoint in neighbouring point numbers.
 
@@ -157,12 +159,14 @@ void Foam::syncTools::syncPointMap
                     }
                 }
 
-                UOPstream toNeighb(procPatch.neighbProcNo(), pBufs);
+                neighbProcs.append(nbrProci);
-                toNeighb << patchInfo;
+                UOPstream toNbr(nbrProci, pBufs);
+                toNbr << patchInfo;
             }
         }
 
-        pBufs.finishedSends();
+        // Limit exchange to involved procs
+        pBufs.finishedNeighbourSends(neighbProcs);
 
         // Receive and combine.
         for (const polyPatch& pp : patches)
@@ -172,8 +176,9 @@ void Foam::syncTools::syncPointMap
             if (ppp && pp.nPoints())
             {
                 const auto& procPatch = *ppp;
+                const label nbrProci = procPatch.neighbProcNo();
 
-                UIPstream fromNbr(procPatch.neighbProcNo(), pBufs);
+                UIPstream fromNbr(nbrProci, pBufs);
                 Map<T> nbrPatchInfo(fromNbr);
 
                 // Transform
@@ -286,11 +291,11 @@ void Foam::syncTools::syncPointMap
         {
             if (Pstream::master())
             {
-                // Receive the edges using shared points from the slave.
+                // Receive the edges using shared points from other procs
-                for (const int slave : Pstream::subProcs())
+                for (const int proci : Pstream::subProcs())
                 {
-                    IPstream fromSlave(Pstream::commsTypes::scheduled, slave);
+                    IPstream fromProc(Pstream::commsTypes::scheduled, proci);
-                    Map<T> nbrValues(fromSlave);
+                    Map<T> nbrValues(fromProc);
 
                     // Merge neighbouring values with my values
                     forAllConstIters(nbrValues, iter)
@@ -304,18 +309,10 @@ void Foam::syncTools::syncPointMap
                         );
                     }
                 }
-
-                // Send back
-                for (const int slave : Pstream::subProcs())
-                {
-                    OPstream toSlave(Pstream::commsTypes::scheduled, slave);
-                    toSlave << sharedPointValues;
-                }
             }
             else
             {
-                // Slave: send to master
+                // Send to master
-                {
                 OPstream toMaster
                 (
                     Pstream::commsTypes::scheduled,
@@ -323,18 +320,10 @@ void Foam::syncTools::syncPointMap
                 );
                 toMaster << sharedPointValues;
             }
-                // Receive merged values
-                {
-                    IPstream fromMaster
-                    (
-                        Pstream::commsTypes::scheduled,
-                        Pstream::masterNo()
-                    );
-                    fromMaster >> sharedPointValues;
-                }
-            }
-        }
 
+            // Broadcast: send merged values to all
+            Pstream::scatter(sharedPointValues);
+        }
 
         // Merge sharedPointValues (keyed on sharedPointAddr) into
         // pointValues (keyed on mesh points).
@@ -381,6 +370,7 @@ void Foam::syncTools::syncEdgeMap
 
     if (Pstream::parRun())
     {
+        DynamicList<label> neighbProcs;
         PstreamBuffers pBufs(Pstream::commsTypes::nonBlocking);
 
         // Send
@@ -391,6 +381,7 @@ void Foam::syncTools::syncEdgeMap
             if (ppp && pp.nEdges())
             {
                 const auto& procPatch = *ppp;
+                const label nbrProci = procPatch.neighbProcNo();
 
                 // Get data per patch edge in neighbouring edge.
 
@@ -413,12 +404,15 @@ void Foam::syncTools::syncEdgeMap
                     }
                 }
 
-                UOPstream toNeighb(procPatch.neighbProcNo(), pBufs);
+                neighbProcs.append(nbrProci);
-                toNeighb << patchInfo;
+                UOPstream toNbr(nbrProci, pBufs);
+                toNbr << patchInfo;
             }
         }
 
-        pBufs.finishedSends();
+        // Limit exchange to involved procs
+        pBufs.finishedNeighbourSends(neighbProcs);
+
 
         // Receive and combine.
         for (const polyPatch& pp : patches)
@@ -640,11 +634,11 @@ void Foam::syncTools::syncEdgeMap
     {
         if (Pstream::master())
         {
-            // Receive the edges using shared points from the slave.
+            // Receive the edges using shared points from other procs
-            for (const int slave : Pstream::subProcs())
+            for (const int proci : Pstream::subProcs())
             {
-                IPstream fromSlave(Pstream::commsTypes::scheduled, slave);
+                IPstream fromProc(Pstream::commsTypes::scheduled, proci);
-                EdgeMap<T> nbrValues(fromSlave);
+                EdgeMap<T> nbrValues(fromProc);
 
                 // Merge neighbouring values with my values
                 forAllConstIters(nbrValues, iter)
@@ -658,13 +652,6 @@ void Foam::syncTools::syncEdgeMap
                     );
                 }
             }
-
-            // Send back
-            for (const int slave : Pstream::subProcs())
-            {
-                OPstream toSlave(Pstream::commsTypes::scheduled, slave);
-                toSlave << sharedEdgeValues;
-            }
         }
         else
         {
@@ -677,16 +664,10 @@ void Foam::syncTools::syncEdgeMap
                 );
                 toMaster << sharedEdgeValues;
             }
-            // Receive merged values
-            {
-                IPstream fromMaster
-                (
-                    Pstream::commsTypes::scheduled,
-                    Pstream::masterNo()
-                );
-                fromMaster >> sharedEdgeValues;
-            }
         }
+
+        // Broadcast: send merged values to all
+        Pstream::scatter(sharedEdgeValues);
     }
 
 
@@ -1121,6 +1102,7 @@ void Foam::syncTools::syncBoundaryFaceList
         }
         else
         {
+            DynamicList<label> neighbProcs;
             PstreamBuffers pBufs(Pstream::commsTypes::nonBlocking);
 
             // Send
@@ -1131,6 +1113,7 @@ void Foam::syncTools::syncBoundaryFaceList
                 if (ppp && pp.size())
                 {
                     const auto& procPatch = *ppp;
+                    const label nbrProci = procPatch.neighbProcNo();
 
                     const SubList<T> fld
                     (
@@ -1139,12 +1122,15 @@ void Foam::syncTools::syncBoundaryFaceList
                         pp.start()-boundaryOffset
                     );
 
-                    UOPstream toNbr(procPatch.neighbProcNo(), pBufs);
+                    neighbProcs.append(nbrProci);
-                    toNbr << fld;;
+                    UOPstream toNbr(nbrProci, pBufs);
+                    toNbr << fld;
                 }
             }
 
-            pBufs.finishedSends();
+            // Limit exchange to involved procs
+            pBufs.finishedNeighbourSends(neighbProcs);
+
 
             // Receive and combine.
             for (const polyPatch& pp : patches)

src/OpenFOAM/meshes/primitiveMesh/PatchTools/PatchToolsGatherAndMerge.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2012-2017 OpenFOAM Foundation
-    Copyright (C) 2019-2020 OpenCFD Ltd.
+    Copyright (C) 2019-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -136,7 +136,7 @@ void Foam::PatchTools::gatherAndMerge
 
     if (Pstream::parRun())
     {
-        // Renumber the setPatch points/faces into unique points
+        // Renumber the points/faces into unique points
         globalPointsPtr = mesh.globalData().mergePoints
         (
             meshPoints,
@@ -147,65 +147,22 @@ void Foam::PatchTools::gatherAndMerge
 
         globalFacesPtr.reset(new globalIndex(localFaces.size()));
 
-        if (Pstream::master())
+        // Renumber faces locally
-        {
-            // Get renumbered local data
-            pointField myPoints(mesh.points(), uniqueMeshPointLabels);
         List<FaceType> myFaces(localFaces);
         for (auto& f : myFaces)
         {
             inplaceRenumber(pointToGlobal, f);
         }
 
-
+        // Can also use
-            mergedFaces.setSize(globalFacesPtr().totalSize());
+        //     UIndirectList<point>(mesh.points(), uniqueMeshPointLabels)
-            mergedPoints.setSize(globalPointsPtr().totalSize());
+        // but favour communication over local memory use
-
+        globalPointsPtr().gather
-            // Insert master data first
-            label pOffset = globalPointsPtr().localStart(Pstream::masterNo());
-            SubList<point>(mergedPoints, myPoints.size(), pOffset) = myPoints;
-
-            label fOffset = globalFacesPtr().localStart(Pstream::masterNo());
-            SubList<FaceType>(mergedFaces, myFaces.size(), fOffset) = myFaces;
-
-
-            // Receive slave ones
-            for (const int slave : Pstream::subProcs())
-            {
-                IPstream fromSlave(Pstream::commsTypes::scheduled, slave);
-
-                pointField slavePoints(fromSlave);
-                List<FaceType> slaveFaces(fromSlave);
-
-                label pOffset = globalPointsPtr().localStart(slave);
-                SubList<point>(mergedPoints, slavePoints.size(), pOffset) =
-                    slavePoints;
-
-                label fOffset = globalFacesPtr().localStart(slave);
-                SubList<FaceType>(mergedFaces, slaveFaces.size(), fOffset) =
-                    slaveFaces;
-            }
-        }
-        else
-        {
-            // Get renumbered local data
-            pointField myPoints(mesh.points(), uniqueMeshPointLabels);
-            List<FaceType> myFaces(localFaces);
-            for (auto& f : myFaces)
-            {
-                inplaceRenumber(pointToGlobal, f);
-            }
-
-            // Construct processor stream with estimate of size. Could
-            // be improved.
-            OPstream toMaster
         (
-                Pstream::commsTypes::scheduled,
+            pointField(mesh.points(), uniqueMeshPointLabels),
-                Pstream::masterNo(),
+            mergedPoints
-                myPoints.byteSize() + 4*sizeof(label)*myFaces.size()
         );
-            toMaster << myPoints << myFaces;
+        globalFacesPtr().gather(myFaces, mergedFaces);
-        }
     }
     else
     {

src/OpenFOAM/parallel/commSchedule/commSchedule.C

@@ -6,6 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2016 OpenFOAM Foundation
+    Copyright (C) 2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -26,8 +27,7 @@ License
 \*---------------------------------------------------------------------------*/
 
 #include "commSchedule.H"
-#include "SortableList.H"
+#include "ListOps.H"
-#include "boolList.H"
 #include "IOstreams.H"
 #include "IOmanip.H"
 #include "StringStream.H"
@@ -41,26 +41,32 @@ namespace Foam
 }
 
 
-// * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
+// * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
 
-Foam::label Foam::commSchedule::outstandingComms
+namespace Foam
+{
+
+// Count the number of outstanding communications for a single processor
+static label outstandingComms
 (
     const labelList& commToSchedule,
     DynamicList<label>& procComms
-) const
+)
 {
     label nOutstanding = 0;
 
-    forAll(procComms, i)
+    for (const label commi : procComms)
     {
-        if (commToSchedule[procComms[i]] == -1)
+        if (commToSchedule[commi] == -1)
         {
-            nOutstanding++;
+            ++nOutstanding;
         }
     }
     return nOutstanding;
 }
 
+} // End namespace Foam
+
 
 // * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
 
@@ -92,9 +98,9 @@ Foam::commSchedule::commSchedule
     }
     // Note: no need to shrink procToComms. Are small.
 
-    if (debug && Pstream::master())
+    if (debug && UPstream::master())
     {
-        Pout<< "commSchedule::commSchedule : Wanted communication:" << endl;
+        Pout<< "commSchedule : Wanted communication:" << endl;
 
         forAll(comms, i)
         {
@@ -106,7 +112,7 @@ Foam::commSchedule::commSchedule
         Pout<< endl;
 
 
-        Pout<< "commSchedule::commSchedule : Schedule:" << endl;
+        Pout<< "commSchedule : Schedule:" << endl;
 
         // Print header. Use buffered output to prevent parallel output messing
         // up.
@@ -191,7 +197,7 @@ Foam::commSchedule::commSchedule
             busy[comms[maxCommI][1]] = true;
         }
 
-        if (debug && Pstream::master())
+        if (debug && UPstream::master())
         {
             label nIterComms = nScheduled-oldNScheduled;
 
@@ -233,14 +239,15 @@ Foam::commSchedule::commSchedule
         iter++;
     }
 
-    if (debug && Pstream::master())
+    if (debug && UPstream::master())
     {
         Pout<< endl;
     }
 
 
-    // Sort commToSchedule and obtain order in comms
+    // Sort commToSchedule to obtain order in comms
-    schedule_ = SortableList<label>(commToSchedule).indices();
+
+    Foam::sortedOrder(commToSchedule, schedule_);
 
     // Sort schedule_ by processor
 
@@ -274,7 +281,7 @@ Foam::commSchedule::commSchedule
         procSchedule_[proc1][nProcScheduled[proc1]++] = commI;
     }
 
-    if (debug && Pstream::master())
+    if (debug && UPstream::master())
     {
         Pout<< "commSchedule::commSchedule : Per processor:" << endl;
 

src/OpenFOAM/parallel/commSchedule/commSchedule.H

@@ -6,6 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2016 OpenFOAM Foundation
+    Copyright (C) 2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -49,10 +50,9 @@ SourceFiles
 
 \*---------------------------------------------------------------------------*/
 
-#ifndef commSchedule_H
+#ifndef Foam_commSchedule_H
-#define commSchedule_H
+#define Foam_commSchedule_H
 
-#include "DynamicList.H"
 #include "labelPair.H"
 #include "labelList.H"
 
@@ -67,7 +67,7 @@ namespace Foam
 
 class commSchedule
 {
-    // Private data
+    // Private Data
 
         //- Order in which input communication has been scheduled
         labelList schedule_;
@@ -75,12 +75,6 @@ class commSchedule
         //- Per processor the order in which communication has been scheduled
         labelListList procSchedule_;
 
-    // Private Member Functions
-
-        //- Count the number of outstanding communications for a single
-        //  processor
-        label outstandingComms(const labelList&, DynamicList<label>&) const;
-
 
 public:
 

src/OpenFOAM/parallel/globalIndex/globalIndex.C

@@ -34,16 +34,16 @@ License
 void Foam::globalIndex::reportOverflowAndExit
 (
     const label idx,
-    const labelUList& localSizes
+    const labelUList& localLens
 )
 {
     FatalErrorInFunction
         << "Overflow : sum of sizes exceeds labelMax ("
         << labelMax << ") after index " << idx;
 
-    if (!localSizes.empty())
+    if (!localLens.empty())
     {
-        FatalError << " of " << flatOutput(localSizes);
+        FatalError << " of " << flatOutput(localLens);
     }
 
     FatalError
@@ -56,13 +56,13 @@ void Foam::globalIndex::reportOverflowAndExit
 Foam::labelList
 Foam::globalIndex::calcOffsets
 (
-    const labelUList& localSizes,
+    const labelUList& localLens,
     const bool checkOverflow
 )
 {
     labelList values;
 
-    const label len = localSizes.size();
+    const label len = localLens.size();
 
     if (len)
     {
@@ -72,11 +72,11 @@ Foam::globalIndex::calcOffsets
         for (label i = 0; i < len; ++i)
         {
             values[i] = start;
-            start += localSizes[i];
+            start += localLens[i];
 
             if (checkOverflow && start < values[i])
             {
-                reportOverflowAndExit(i, localSizes);
+                reportOverflowAndExit(i, localLens);
             }
         }
         values[len] = start;
@@ -89,13 +89,13 @@ Foam::globalIndex::calcOffsets
 Foam::List<Foam::labelRange>
 Foam::globalIndex::calcRanges
 (
-    const labelUList& localSizes,
+    const labelUList& localLens,
     const bool checkOverflow
 )
 {
     List<labelRange> values;
 
-    const label len = localSizes.size();
+    const label len = localLens.size();
 
     if (len)
     {
@@ -104,12 +104,12 @@ Foam::globalIndex::calcRanges
         label start = 0;
         for (label i = 0; i < len; ++i)
         {
-            values[i].reset(start, localSizes[i]);
+            values[i].reset(start, localLens[i]);
-            start += localSizes[i];
+            start += localLens[i];
 
             if (checkOverflow && start < values[i].start())
             {
-                reportOverflowAndExit(i, localSizes);
+                reportOverflowAndExit(i, localLens);
             }
         }
     }
@@ -221,16 +221,16 @@ void Foam::globalIndex::reset
         // Seed with localSize, zero elsewhere (for non-parallel branch)
         // NB: can consider UPstream::listGatherValues
 
-        labelList localSizes(len, Zero);
+        labelList localLens(len, Zero);
-        localSizes[Pstream::myProcNo(comm)] = localSize;
+        localLens[Pstream::myProcNo(comm)] = localSize;
 
         if (parallel)
         {
-            Pstream::gatherList(localSizes, tag, comm);
+            Pstream::gatherList(localLens, tag, comm);
-            Pstream::scatterList(localSizes, tag, comm);
+            Pstream::scatterList(localLens, tag, comm);
         }
 
-        reset(localSizes, true);  // checkOverflow = true
+        reset(localLens, true);  // checkOverflow = true
     }
     else
     {
@@ -242,11 +242,11 @@ void Foam::globalIndex::reset
 
 void Foam::globalIndex::reset
 (
-    const labelUList& localSizes,
+    const labelUList& localLens,
     const bool checkOverflow
 )
 {
-    const label len = localSizes.size();
+    const label len = localLens.size();
 
     if (len)
     {
@@ -256,11 +256,11 @@ void Foam::globalIndex::reset
         for (label i = 0; i < len; ++i)
         {
             offsets_[i] = start;
-            start += localSizes[i];
+            start += localLens[i];
 
             if (checkOverflow && start < offsets_[i])
             {
-                reportOverflowAndExit(i, localSizes);
+                reportOverflowAndExit(i, localLens);
             }
         }
         offsets_[len] = start;
@@ -290,7 +290,7 @@ void Foam::globalIndex::setLocalSize(const label proci, const label len)
 }
 
 
-Foam::labelList Foam::globalIndex::sizes() const
+Foam::labelList Foam::globalIndex::localSizes() const
 {
     labelList values;
 

src/OpenFOAM/parallel/globalIndex/globalIndex.H

@@ -89,7 +89,7 @@ class globalIndex
         static void reportOverflowAndExit
         (
             const label idx,
-            const labelUList& localSizes = labelUList::null()
+            const labelUList& localLens = labelUList::null()
         );
 
 public:
@@ -178,12 +178,15 @@ public:
         //- Global sum of localSizes.
         inline label totalSize() const;
 
-        //- The local sizes
+        //- The local sizes. Same as localSizes()
-        labelList sizes() const;
+        inline labelList sizes() const;
 
         //- The local starts
         inline const labelUList localStarts() const;
 
+        //- The local sizes
+        labelList localSizes() const;
+
         //- Global max of localSizes
         inline label maxSize() const;
 
@@ -276,7 +279,7 @@ public:
             inline labelList toGlobal(const labelUList& labels) const;
 
             //- From local to global index (inplace)
-            inline void inplaceToGlobal(labelList& labels) const;
+            inline void inplaceToGlobal(labelUList& labels) const;
 
             //- From global to local on current processor.
             //  FatalError if not on local processor.
@@ -314,7 +317,7 @@ public:
             inline void inplaceToGlobal
             (
                 const label proci,
-                labelList& labels
+                labelUList& labels
             ) const;
 
 
@@ -405,7 +408,7 @@ public:
         //- with optional check for label overflow
         static labelList calcOffsets
         (
-            const labelUList& localSizes,
+            const labelUList& localLens,
             const bool checkOverflow = false
         );
 
@@ -422,7 +425,7 @@ public:
         //- with optional check for label overflow
         static List<labelRange> calcRanges
         (
-            const labelUList& localSizes,
+            const labelUList& localLens,
             const bool checkOverflow = false
         );
 
@@ -774,8 +777,21 @@ public:
             template<class Type>
             void scatter
             (
-                const UList<Type>& allFld,
+                const UList<Type>& allData,
-                UList<Type>& fld,
+                UList<Type>& localData,
+                const int tag = UPstream::msgType(),
+                const Pstream::commsTypes = Pstream::commsTypes::nonBlocking,
+                const label comm = UPstream::worldComm  //!< communicator
+            ) const;
+
+            //- Distribute data in processor order
+            //- (in serial: performs a simple copy).
+            //  Communication with default/specified communicator, message tag.
+            //  \note the globalIndex offsets needed on master only.
+            template<class Type, class OutputContainer = List<Type>>
+            OutputContainer scatter
+            (
+                const UList<Type>& allData,
                 const int tag = UPstream::msgType(),
                 const Pstream::commsTypes = Pstream::commsTypes::nonBlocking,
                 const label comm = UPstream::worldComm  //!< communicator

src/OpenFOAM/parallel/globalIndex/globalIndexI.H

@@ -142,6 +142,12 @@ inline Foam::label Foam::globalIndex::size() const
 }
 
 
+inline Foam::labelList Foam::globalIndex::sizes() const
+{
+    return localSizes();
+}
+
+
 inline Foam::label Foam::globalIndex::nProcs() const noexcept
 {
     const label len = (offsets_.size() - 1);
@@ -289,7 +295,7 @@ inline Foam::labelList Foam::globalIndex::toGlobal
 inline void Foam::globalIndex::inplaceToGlobal
 (
     const label proci,
-    labelList& labels
+    labelUList& labels
 ) const
 {
     const label off = offsets_[proci];
@@ -301,7 +307,7 @@ inline void Foam::globalIndex::inplaceToGlobal
 }
 
 
-inline void Foam::globalIndex::inplaceToGlobal(labelList& labels) const
+inline void Foam::globalIndex::inplaceToGlobal(labelUList& labels) const
 {
     inplaceToGlobal(Pstream::myProcNo(), labels);
 }

src/OpenFOAM/parallel/globalIndex/globalIndexTemplates.C

@@ -73,22 +73,22 @@ void Foam::globalIndex::gather
     const UList<Type>& fld,
     List<Type>& allFld,
     const int tag,
-    const Pstream::commsTypes commsType
+    const Pstream::commsTypes preferredCommsType
 )
 {
     // low-level: no parRun guard
 
-    if
+    // Automatically change from nonBlocking to scheduled for
+    // non-contiguous data.
+    const UPstream::commsTypes commsType =
+    (
         (
             !is_contiguous<Type>::value
-     && commsType == Pstream::commsTypes::nonBlocking
+         && UPstream::commsTypes::nonBlocking == preferredCommsType
         )
-    {
+      ? UPstream::commsTypes::scheduled
-        FatalErrorInFunction
+      : preferredCommsType
-            << "Cannot use nonBlocking with non-contiguous data"
+    );
-            << exit(FatalError);
-        // Could also warn and change to scheduled etc...
-    }
 
     const label startOfRequests = UPstream::nRequests();
 
@@ -126,14 +126,7 @@ void Foam::globalIndex::gather
             }
             else
             {
-                IPstream fromProc
+                IPstream fromProc(commsType, procIDs[i], 0, tag, comm);
-                (
-                    commsType,
-                    procIDs[i],
-                    0,
-                    tag,
-                    comm
-                );
                 fromProc >> procSlot;
             }
         }
@@ -158,14 +151,7 @@ void Foam::globalIndex::gather
         }
         else
         {
-            OPstream toMaster
+            OPstream toMaster(commsType, procIDs[0], 0, tag, comm);
-            (
-                commsType,
-                procIDs[0],
-                0,
-                tag,
-                comm
-            );
             toMaster << fld;
         }
     }
@@ -187,18 +173,22 @@ void Foam::globalIndex::gather
     const IndirectListBase<Type, Addr>& fld,
     List<Type>& allFld,
     const int tag,
-    const Pstream::commsTypes commsType
+    const Pstream::commsTypes preferredCommsType
 )
 {
     // low-level: no parRun guard
 
-    if (commsType == Pstream::commsTypes::nonBlocking)
+    // Automatically change from nonBlocking to scheduled for
-    {
+    // non-contiguous data.
-        WarningInFunction
+    const UPstream::commsTypes commsType =
-            << "Cannot use nonBlocking with indirect list of data"
+    (
-            << exit(FatalError);
+        (
-        // Could also warn and change to scheduled etc...
+            !is_contiguous<Type>::value
-    }
+         && UPstream::commsTypes::nonBlocking == preferredCommsType
+        )
+      ? UPstream::commsTypes::scheduled
+      : preferredCommsType
+    );
 
     if (Pstream::myProcNo(comm) == procIDs[0])
     {
@@ -224,14 +214,7 @@ void Foam::globalIndex::gather
             }
             else
             {
-                IPstream fromProc
+                IPstream fromProc(commsType, procIDs[i], 0, tag, comm);
-                (
-                    commsType,
-                    procIDs[i],
-                    0,
-                    tag,
-                    comm
-                );
                 fromProc >> procSlot;
             }
         }
@@ -244,14 +227,7 @@ void Foam::globalIndex::gather
         }
         else
         {
-            OPstream toMaster
+            OPstream toMaster(commsType, procIDs[0], 0, tag, comm);
-            (
-                commsType,
-                procIDs[0],
-                0,
-                tag,
-                comm
-            );
             toMaster << fld;
         }
     }
@@ -692,20 +668,22 @@ void Foam::globalIndex::scatter
     const UList<Type>& allFld,
     UList<Type>& fld,
     const int tag,
-    const Pstream::commsTypes commsType
+    const Pstream::commsTypes preferredCommsType
 )
 {
-    if
+    // low-level: no parRun guard
+
+    // Automatically change from nonBlocking to scheduled for
+    // non-contiguous data.
+    const UPstream::commsTypes commsType =
+    (
         (
             !is_contiguous<Type>::value
-     && commsType == Pstream::commsTypes::nonBlocking
+         && UPstream::commsTypes::nonBlocking == preferredCommsType
         )
-    {
+      ? UPstream::commsTypes::scheduled
-        FatalErrorInFunction
+      : preferredCommsType
-            << "Cannot use nonBlocking with non-contiguous data"
+    );
-            << exit(FatalError);
-        // Could also warn and change to scheduled etc...
-    }
 
     // FUTURE:
     // could decide which procs will receive data and use mpiScatter
@@ -715,18 +693,15 @@ void Foam::globalIndex::scatter
 
     if (Pstream::myProcNo(comm) == procIDs[0])
     {
-        const SubList<Type> localSlot(allFld, off[1]-off[0], off[0]);
-
-        if (!localSlot.empty())
-        {
-            fld.deepCopy(localSlot);
-        }
-
         for (label i = 1; i < procIDs.size(); ++i)
         {
             const SubList<Type> procSlot(allFld, off[i+1]-off[i], off[i]);
 
-            if (is_contiguous<Type>::value)
+            if (procSlot.empty())
+            {
+                // Nothing to do
+            }
+            else if (is_contiguous<Type>::value)
             {
                 OPstream::write
                 (
@@ -740,21 +715,30 @@ void Foam::globalIndex::scatter
             }
             else
             {
-                OPstream toProc
+                OPstream toProc(commsType, procIDs[i], 0, tag, comm);
-                (
-                    commsType,
-                    procIDs[i],
-                    0,
-                    tag,
-                    comm
-                );
                 toProc << procSlot;
             }
         }
+
+        // Assign my local data - respect offset information
+        // so that we can request 0 entries to be copied.
+        // Also handle the case where we have a slice of the full
+        // list.
+
+        SubList<Type>(fld, off[1]-off[0]) =
+            SubList<Type>(allFld, off[1]-off[0], off[0]);
     }
     else
     {
-        if (is_contiguous<Type>::value)
+        // Note: we are receiving into UList, so sizes MUST match or we
+        // have a problem. Can therefore reasonably assume that a zero-sized
+        // send matches a zero-sized receive, and we can skip that.
+
+        if (fld.empty())
+        {
+            // Nothing to do
+        }
+        else if (is_contiguous<Type>::value)
         {
             IPstream::read
             (
@@ -768,14 +752,7 @@ void Foam::globalIndex::scatter
         }
         else
         {
-            IPstream fromMaster
+            IPstream fromMaster(commsType, procIDs[0], 0, tag, comm);
-            (
-                commsType,
-                procIDs[0],
-                0,
-                tag,
-                comm
-            );
             fromMaster >> fld;
         }
     }
@@ -791,26 +768,64 @@ void Foam::globalIndex::scatter
 template<class Type>
 void Foam::globalIndex::scatter
 (
-    const UList<Type>& allFld,
+    const UList<Type>& allData,
-    UList<Type>& fld,
+    UList<Type>& localData,
     const int tag,
     const Pstream::commsTypes commsType,
     const label comm
 ) const
 {
-    // TBD: protection and special handling for serial?
+    if (UPstream::parRun())
     {
         scatter
         (
             offsets_,  // needed on master only
             comm,
             UPstream::procID(comm),
-            allFld,
+            allData,
-            fld,
+            localData,
             tag,
             commsType
         );
     }
+    else
+    {
+        // Serial: direct copy
+        // - fails miserably if incorrectly dimensioned!
+        localData.deepCopy(allData);
+    }
+}
+
+
+template<class Type, class OutputContainer>
+OutputContainer Foam::globalIndex::scatter
+(
+    const UList<Type>& allData,
+    const int tag,
+    const Pstream::commsTypes commsType,
+    const label comm
+) const
+{
+    if (UPstream::parRun())
+    {
+        // The globalIndex might be correct on master only,
+        // so scatter local sizes to ensure consistency
+
+        const label localLen
+        (
+            UPstream::listScatterValues<label>(this->localSizes(), comm)
+        );
+
+        OutputContainer localData(localLen);
+        this->scatter(allData, localData, tag, commsType, comm);
+
+        return localData;
+    }
+    else
+    {
+        // Serial: direct copy
+        return OutputContainer(allData);
+    }
 }
 
 
@@ -856,15 +871,14 @@ void Foam::globalIndex::get
             UOPstream os(proci, sendBufs);
             os << localIDs;
         }
-        labelList recvSizes;
+        sendBufs.finishedSends();
-        sendBufs.finishedSends(recvSizes);
 
 
         PstreamBuffers returnBufs(Pstream::commsTypes::nonBlocking, tag, comm);
 
-        forAll(recvSizes, proci)
+        for (const int proci : sendBufs.allProcs())
         {
-            if (recvSizes[proci])
+            if (sendBufs.hasRecvData(proci))
             {
                 UIPstream is(proci, sendBufs);
                 labelList localIDs(is);

src/Pstream/dummy/Make/files

@@ -1,5 +1,10 @@
 UPstream.C
-UIPread.C
+UPstreamBroadcast.C
-UOPwrite.C
+UPstreamReduce.C
+
+UIPstreamRead.C
+UOPstreamWrite.C
+UIPBstreamRead.C
+UOPBstreamWrite.C
 
 LIB = $(FOAM_LIBBIN)/dummy/libPstream

src/Pstream/dummy/UIPBstreamRead.C

@@ -5,8 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2011-2013 OpenFOAM Foundation
+    Copyright (C) 2022 OpenCFD Ltd.
-    Copyright (C) 2021 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -26,32 +25,31 @@ License
 
 \*---------------------------------------------------------------------------*/
 
-#include "OPstream.H"
+#include "UIPstream.H"
 
-// * * * * * * * * * * * * * * * * Constructor * * * * * * * * * * * * * * * //
+// * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
 
-Foam::OPstream::OPstream
+void Foam::UIPBstream::bufferIPCrecv()
+{
+    NotImplemented;
+}
+
+
+// * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
+
+Foam::label Foam::UIPBstream::read
 (
     const commsTypes commsType,
-    const int toProcNo,
+    const int rootProcNo,
-    const label bufSize,
+    char* buf,
+    const std::streamsize bufSize,
     const int tag,
-    const label comm,
+    const label communicator
-    IOstreamOption::streamFormat fmt
 )
-:
+{
-    Pstream(commsType, bufSize),
+    NotImplemented;
-    UOPstream
+    return 0;
-    (
+}
-        commsType,
-        toProcNo,
-        Pstream::transferBuf_,
-        tag,
-        comm,
-        true,  // sendAtDestruct
-        fmt
-    )
-{}
 
 
 // ************************************************************************* //

src/Pstream/dummy/UIPstreamRead.C

@@ -5,8 +5,8 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2011-2013 OpenFOAM Foundation
+    Copyright (C) 2011-2015 OpenFOAM Foundation
-    Copyright (C) 2021 OpenCFD Ltd.
+    Copyright (C) 2021-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -26,34 +26,31 @@ License
 
 \*---------------------------------------------------------------------------*/
 
-#include "IPstream.H"
+#include "UIPstream.H"
 
-// * * * * * * * * * * * * * * * * Constructor * * * * * * * * * * * * * * * //
+// * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
 
-Foam::IPstream::IPstream
+void Foam::UIPstream::bufferIPCrecv()
+{
+    NotImplemented;
+}
+
+
+// * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
+
+Foam::label Foam::UIPstream::read
 (
     const commsTypes commsType,
     const int fromProcNo,
-    const label bufSize,
+    char* buf,
+    const std::streamsize bufSize,
     const int tag,
-    const label comm,
+    const label communicator
-    IOstreamOption::streamFormat fmt
 )
-:
+{
-    Pstream(commsType, bufSize),
+    NotImplemented;
-    UIPstream
+    return 0;
-    (
+}
-        commsType,
-        fromProcNo,
-        Pstream::transferBuf_,
-        transferBufPosition_,
-        tag,
-        comm,
-        false,  // Do not clear Pstream::transferBuf_ if at end
-        fmt
-    ),
-    transferBufPosition_(0)
-{}
 
 
 // ************************************************************************* //

src/Pstream/dummy/UOPBstreamWrite.C

@@ -0,0 +1,56 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | www.openfoam.com
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+    Copyright (C) 2022 OpenCFD Ltd.
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "UOPstream.H"
+
+// * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
+
+bool Foam::UOPBstream::bufferIPCsend()
+{
+    NotImplemented;
+    return false;
+}
+
+
+// * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
+
+bool Foam::UOPBstream::write
+(
+    const commsTypes commsType,
+    const int rootProcNo,
+    const char* buf,
+    const std::streamsize bufSize,
+    const int tag,
+    const label communicator
+)
+{
+    NotImplemented;
+    return false;
+}
+
+
+// ************************************************************************* //

src/Pstream/dummy/UOPstreamWrite.C

@@ -6,6 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2015 OpenFOAM Foundation
+    Copyright (C) 2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -23,13 +24,19 @@ License
     You should have received a copy of the GNU General Public License
     along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
 
-Description
-    Write primitive and binary block from OPstream
-
 \*---------------------------------------------------------------------------*/
 
 #include "UOPstream.H"
 
+// * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
+
+bool Foam::UOPstream::bufferIPCsend()
+{
+    NotImplemented;
+    return false;
+}
+
+
 // * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
 
 bool Foam::UOPstream::write
@@ -43,7 +50,6 @@ bool Foam::UOPstream::write
 )
 {
     NotImplemented;
-
     return false;
 }
 

src/Pstream/dummy/UPstream.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2018 OpenFOAM Foundation
-    Copyright (C) 2016-2021 OpenCFD Ltd.
+    Copyright (C) 2016-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -27,7 +27,6 @@ License
 \*---------------------------------------------------------------------------*/
 
 #include "Pstream.H"
-#include "PstreamReduceOps.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
@@ -74,99 +73,7 @@ void Foam::UPstream::abort()
 }
 
 
-void Foam::reduce(scalar&, const sumOp<scalar>&, const int, const label)
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-{}
-
-
-void Foam::reduce(scalar&, const minOp<scalar>&, const int, const label)
-{}
-
-
-void Foam::reduce(vector2D&, const sumOp<vector2D>&, const int, const label)
-{}
-
-
-void Foam::sumReduce
-(
-    scalar&,
-    label&,
-    const int,
-    const label
-)
-{}
-
-
-void Foam::reduce(scalar&, const sumOp<scalar>&, const int, const label, label&)
-{}
-
-
-void Foam::reduce
-(
-    scalar[],
-    const int,
-    const sumOp<scalar>&,
-    const int,
-    const label,
-    label&
-)
-{}
-
-
-#if defined(WM_SPDP)
-void Foam::reduce
-(
-    solveScalar& Value,
-    const sumOp<solveScalar>& bop,
-    const int tag,
-    const label comm
-)
-{}
-void Foam::reduce
-(
-    solveScalar& Value,
-    const minOp<solveScalar>& bop,
-    const int tag,
-    const label comm
-)
-{}
-void Foam::reduce
-(
-    Vector2D<solveScalar>& Value,
-    const sumOp<Vector2D<solveScalar>>& bop,
-    const int tag,
-    const label comm
-)
-{}
-void Foam::sumReduce
-(
-    solveScalar& Value,
-    label& Count,
-    const int tag,
-    const label comm
-)
-{}
-void Foam::reduce
-(
-    solveScalar& Value,
-    const sumOp<solveScalar>& bop,
-    const int tag,
-    const label comm,
-    label& request
-)
-{}
-void Foam::reduce
-(
-    solveScalar[],
-    const int,
-    const sumOp<solveScalar>&,
-    const int,
-    const label,
-    label&
-)
-{}
-#endif
-
-
 
 void Foam::UPstream::allToAll
 (

src/Pstream/dummy/UPstreamBroadcast.C

@@ -0,0 +1,45 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | www.openfoam.com
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+    Copyright (C) 2022 OpenCFD Ltd.
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "UPstream.H"
+
+// * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
+
+bool Foam::UPstream::broadcast
+(
+    char* buf,
+    const std::streamsize bufSize,
+    const label communicator,
+    const int rootProcNo
+)
+{
+    // Nothing to do - ignore
+    return true;
+}
+
+
+// ************************************************************************* //

src/Pstream/dummy/UPstreamReduce.C

@@ -0,0 +1,153 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | www.openfoam.com
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+    Copyright (C) 2022 OpenCFD Ltd.
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "Pstream.H"
+#include "PstreamReduceOps.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+// Specialisations for bool
+
+void Foam::reduce
+(
+    bool& value,
+    const andOp<bool>&,
+    const int tag,
+    const label comm
+)
+{}
+
+void Foam::reduce
+(
+    bool& value,
+    const orOp<bool>&,
+    const int tag,
+    const label comm
+)
+{}
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+// Specialisations for common reduction types
+
+#undef  Pstream_CommonReductions
+#define Pstream_CommonReductions(Native)                                      \
+                                                                              \
+void Foam::reduce                                                             \
+(                                                                             \
+    Native& value,                                                            \
+    const minOp<Native>&,                                                     \
+    const int tag,                                                            \
+    const label comm                                                          \
+)                                                                             \
+{}                                                                            \
+                                                                              \
+void Foam::reduce                                                             \
+(                                                                             \
+    Native& value,                                                            \
+    const maxOp<Native>&,                                                     \
+    const int tag,                                                            \
+    const label comm                                                          \
+)                                                                             \
+{}                                                                            \
+                                                                              \
+void Foam::reduce                                                             \
+(                                                                             \
+    Native& value,                                                            \
+    const sumOp<Native>&,                                                     \
+    const int tag,                                                            \
+    const label comm                                                          \
+)                                                                             \
+{}                                                                            \
+                                                                              \
+void Foam::reduce                                                             \
+(                                                                             \
+    Native values[],                                                          \
+    const int size,                                                           \
+    const sumOp<Native>&,                                                     \
+    const int tag,                                                            \
+    const label comm                                                          \
+)                                                                             \
+{}
+
+
+Pstream_CommonReductions(int32_t);
+Pstream_CommonReductions(int64_t);
+Pstream_CommonReductions(uint32_t);
+Pstream_CommonReductions(uint64_t);
+Pstream_CommonReductions(float);
+Pstream_CommonReductions(double);
+
+#undef Pstream_CommonReductions
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+// Specialisations for floating-point types
+
+#undef  Pstream_FloatReductions
+#define Pstream_FloatReductions(Native)                                       \
+                                                                              \
+void Foam::sumReduce                                                          \
+(                                                                             \
+    Native& value,                                                            \
+    label& count,                                                             \
+    const int tag,                                                            \
+    const label comm                                                          \
+)                                                                             \
+{}                                                                            \
+                                                                              \
+void Foam::reduce                                                             \
+(                                                                             \
+    Native& value,                                                            \
+    const sumOp<Native>&,                                                     \
+    const int tag,                                                            \
+    const label comm,                                                         \
+    label& requestID                                                          \
+)                                                                             \
+{}                                                                            \
+                                                                              \
+void Foam::reduce                                                             \
+(                                                                             \
+    Native values[],                                                          \
+    const int size,                                                           \
+    const sumOp<Native>&,                                                     \
+    const int tag,                                                            \
+    const label comm,                                                         \
+    label& requestID                                                          \
+)                                                                             \
+{}
+
+
+Pstream_FloatReductions(float);
+Pstream_FloatReductions(double);
+
+#undef Pstream_FloatReductions
+
+
+// ************************************************************************* //

src/Pstream/mpi/Make/files

@@ -1,6 +1,11 @@
-UOPwrite.C
-UIPread.C
-UPstream.C
 PstreamGlobals.C
+UPstream.C
+UPstreamBroadcast.C
+UPstreamReduce.C
+
+UIPstreamRead.C
+UOPstreamWrite.C
+UIPBstreamRead.C
+UOPBstreamWrite.C
 
 LIB = $(FOAM_MPI_LIBBIN)/libPstream

src/Pstream/mpi/PstreamGlobals.H

@@ -35,8 +35,8 @@ SourceFiles
 
 \*---------------------------------------------------------------------------*/
 
-#ifndef PstreamGlobals_H
+#ifndef Foam_PstreamGlobals_H
-#define PstreamGlobals_H
+#define Foam_PstreamGlobals_H
 
 #include "DynamicList.H"
 #include <mpi.h>
@@ -62,14 +62,12 @@ extern DynamicList<int> freedTags_;
 extern DynamicList<MPI_Comm> MPICommunicators_;
 extern DynamicList<MPI_Group> MPIGroups_;
 
-
 void checkCommunicator(const label comm, const label toProcNo);
 
-};
-
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
+} // End namespace PstreamGlobals
 } // End namespace Foam
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //

src/Pstream/mpi/UIPBstreamRead.C

@@ -0,0 +1,139 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | www.openfoam.com
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+    Copyright (C) 2022 OpenCFD Ltd.
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "UIPstream.H"
+#include "PstreamGlobals.H"
+#include "IOstreams.H"
+
+// * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
+
+void Foam::UIPBstream::bufferIPCrecv()
+{
+    // Uses double broadcast. Symmetric with UOPBstream::bufferIPCsend()
+    // 1. for the data size
+    // 2. for the data itself
+
+    // Expected message size, similar to MPI_Probe
+    // Same type must be expected in UOPBstream::bufferIPCsend()
+    label bufSize(0);
+
+    // Broadcast #1 - data size
+    if
+    (
+        !UPstream::broadcast
+        (
+            reinterpret_cast<char*>(&bufSize),
+            sizeof(label),
+            comm_,
+            fromProcNo_  //< is actually rootProcNo
+        )
+    )
+    {
+        FatalErrorInFunction
+            << "MPI_Bcast failure receiving buffer size" << nl
+            << Foam::abort(FatalError);
+    }
+
+    if (debug)
+    {
+        Pout<< "UOPBstream IPC read buffer :"
+            << " root:" << fromProcNo_
+            << " comm:" << comm_
+            << " probed size:" << bufSize
+            << " wanted size:" << recvBuf_.capacity()
+            << Foam::endl;
+    }
+
+    // No buffer size allocated/specified
+    if (!recvBuf_.capacity())
+    {
+        recvBuf_.resize(bufSize);
+    }
+
+    // This is the only real information we can trust
+    messageSize_ = bufSize;
+
+    // Broadcast #2 - data content
+    // - skip if there is no data to receive
+
+    if (messageSize_)
+    {
+        if
+        (
+            !UPstream::broadcast
+            (
+                recvBuf_.data(),
+                messageSize_,  // same as bufSize
+                comm_,
+                fromProcNo_  //< is actually rootProcNo
+            )
+        )
+        {
+            FatalErrorInFunction
+                << "MPI_Bcast failure receiving buffer data:" << bufSize << nl
+                << Foam::abort(FatalError);
+        }
+    }
+
+    // Set addressed size. Leave actual allocated memory intact.
+    recvBuf_.resize(messageSize_);
+
+    if (!messageSize_)
+    {
+        setEof();
+    }
+}
+
+
+// * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
+
+Foam::label Foam::UIPBstream::read
+(
+    const commsTypes commsType,
+    const int rootProcNo,
+    char* buf,
+    const std::streamsize bufSize,
+    const int tag,
+    const label comm
+)
+{
+    if
+    (
+        !UPstream::broadcast(buf, bufSize, comm, rootProcNo)
+    )
+    {
+        FatalErrorInFunction
+            << "MPI_Bcast failure receiving data:" << label(bufSize) << nl
+            << Foam::abort(FatalError);
+        return 0;
+    }
+
+    return bufSize;
+}
+
+
+// ************************************************************************* //

src/Pstream/mpi/UIPstreamRead.C

@@ -24,9 +24,6 @@ License
     You should have received a copy of the GNU General Public License
     along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
 
-Description
-    Read from UIPstream
-
 \*---------------------------------------------------------------------------*/
 
 #include "UIPstream.H"
@@ -36,43 +33,15 @@ Description
 
 #include <mpi.h>
 
-// * * * * * * * * * * * * * * * * Constructor * * * * * * * * * * * * * * * //
+// * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
 
-Foam::UIPstream::UIPstream
+void Foam::UIPstream::bufferIPCrecv()
-(
-    const commsTypes commsType,
-    const int fromProcNo,
-    DynamicList<char>& receiveBuf,
-    label& receiveBufPosition,
-    const int tag,
-    const label comm,
-    const bool clearAtEnd,
-    IOstreamOption::streamFormat fmt
-)
-:
-    UPstream(commsType),
-    Istream(fmt, IOstreamOption::currentVersion),
-    fromProcNo_(fromProcNo),
-    recvBuf_(receiveBuf),
-    recvBufPos_(receiveBufPosition),
-    tag_(tag),
-    comm_(comm),
-    clearAtEnd_(clearAtEnd),
-    messageSize_(0)
-{
-    setOpened();
-    setGood();
-
-    if (commsType == commsTypes::nonBlocking)
-    {
-        // Message is already received into buffer
-    }
-    else
 {
+    // Called by constructor
     if (debug)
     {
-            Pout<< "UIPstream::UIPstream :"
+        Pout<< "UIPstream IPC read buffer :"
-                << " read from:" << fromProcNo
+            << " from:" << fromProcNo_
             << " tag:" << tag_ << " comm:" << comm_
             << " wanted size:" << recvBuf_.capacity()
             << Foam::endl;
@@ -106,108 +75,6 @@ Foam::UIPstream::UIPstream
         }
     }
 
-        messageSize_ = UIPstream::read
-        (
-            commsType,
-            fromProcNo_,
-            recvBuf_.data(),
-            recvBuf_.capacity(),
-            tag_,
-            comm_
-        );
-
-        // Set addressed size. Leave actual allocated memory intact.
-        recvBuf_.resize(messageSize_);
-
-        if (!messageSize_)
-        {
-            setEof();
-        }
-    }
-}
-
-
-Foam::UIPstream::UIPstream(const int fromProcNo, PstreamBuffers& buffers)
-:
-    UPstream(buffers.commsType_),
-    Istream(buffers.format_, IOstreamOption::currentVersion),
-    fromProcNo_(fromProcNo),
-    recvBuf_(buffers.recvBuf_[fromProcNo]),
-    recvBufPos_(buffers.recvBufPos_[fromProcNo]),
-    tag_(buffers.tag_),
-    comm_(buffers.comm_),
-    clearAtEnd_(true),
-    messageSize_(0)
-{
-    if
-    (
-        commsType() != UPstream::commsTypes::scheduled
-     && !buffers.finishedSendsCalled_
-    )
-    {
-        FatalErrorInFunction
-            << "PstreamBuffers::finishedSends() never called." << endl
-            << "Please call PstreamBuffers::finishedSends() after doing"
-            << " all your sends (using UOPstream) and before doing any"
-            << " receives (using UIPstream)" << Foam::exit(FatalError);
-    }
-
-    setOpened();
-    setGood();
-
-    if (commsType() == commsTypes::nonBlocking)
-    {
-        // Message is already received into buffer
-        messageSize_ = recvBuf_.size();
-
-        if (debug)
-        {
-            Pout<< "UIPstream::UIPstream PstreamBuffers :"
-                << " fromProcNo:" << fromProcNo
-                << " tag:" << tag_ << " comm:" << comm_
-                << " receive buffer size:" << messageSize_
-                << Foam::endl;
-        }
-    }
-    else
-    {
-        if (debug)
-        {
-            Pout<< "UIPstream::UIPstream PstreamBuffers :"
-                << " read from:" << fromProcNo
-                << " tag:" << tag_ << " comm:" << comm_
-                << " wanted size:" << recvBuf_.capacity()
-                << Foam::endl;
-        }
-
-        // No buffer size allocated/specified - probe size of incoming message
-        if (!recvBuf_.capacity())
-        {
-            profilingPstream::beginTiming();
-
-            MPI_Status status;
-
-            MPI_Probe
-            (
-                fromProcNo_,
-                tag_,
-                PstreamGlobals::MPICommunicators_[comm_],
-                &status
-            );
-            MPI_Get_count(&status, MPI_BYTE, &messageSize_);
-
-            // Assume these are from gathers ...
-            profilingPstream::addGatherTime();
-
-            recvBuf_.resize(messageSize_);
-
-            if (debug)
-            {
-                Pout<< "UIPstream::UIPstream PstreamBuffers : probed size:"
-                    << messageSize_ << Foam::endl;
-            }
-        }
-
     messageSize_ = UIPstream::read
     (
         commsType(),
@@ -226,7 +93,6 @@ Foam::UIPstream::UIPstream(const int fromProcNo, PstreamBuffers& buffers)
         setEof();
     }
 }
-}
 
 
 // * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
@@ -287,7 +153,6 @@ Foam::label Foam::UIPstream::read
             FatalErrorInFunction
                 << "MPI_Recv cannot receive incoming message"
                 << Foam::abort(FatalError);
-
             return 0;
         }
 
@@ -336,7 +201,7 @@ Foam::label Foam::UIPstream::read
         )
         {
             FatalErrorInFunction
-                << "MPI_Recv cannot start non-blocking receive"
+                << "MPI_Irecv cannot start non-blocking receive"
                 << Foam::abort(FatalError);
 
             return 0;

src/Pstream/mpi/UOPBstreamWrite.C

@@ -0,0 +1,118 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | www.openfoam.com
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+    Copyright (C) 2022 OpenCFD Ltd.
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "UOPstream.H"
+#include "PstreamGlobals.H"
+
+#include <mpi.h>
+
+// * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
+
+bool Foam::UOPBstream::bufferIPCsend()
+{
+    // Uses double broadcast
+    // 1. for the data size
+    // 2. for the data itself
+    // With this information, can determine and resize receive buffer
+
+    PstreamGlobals::checkCommunicator(comm_, toProcNo_);
+
+    // Same type must be expected in UIPBstream::bufferIPCrecv()
+    label bufSize(sendBuf_.size());
+
+    // Broadcast #1 - data size
+    if
+    (
+        !UPstream::broadcast
+        (
+            reinterpret_cast<char*>(&bufSize),
+            sizeof(label),
+            comm_,
+            toProcNo_  //< is actually rootProcNo
+        )
+    )
+    {
+        FatalErrorInFunction
+            << "MPI_Bcast failure sending buffer size:" << bufSize << nl
+            << Foam::abort(FatalError);
+        return false;
+    }
+
+    // Broadcast #2 - data content
+    // - skip if there is no data to send
+    if (bufSize)
+    {
+        if
+        (
+            !UPstream::broadcast
+            (
+                sendBuf_.data(),
+                sendBuf_.size(),  // same as bufSize
+                comm_,
+                toProcNo_  //< is actually rootProcNo
+            )
+        )
+        {
+            FatalErrorInFunction
+                << "MPI_Bcast failure sending buffer data:" << bufSize << nl
+                << Foam::abort(FatalError);
+            return false;
+        }
+    }
+
+    return true;
+}
+
+
+// * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
+
+bool Foam::UOPBstream::write
+(
+    const commsTypes commsType,  /* unused */
+    const int rootProcNo,
+    const char* buf,
+    const std::streamsize bufSize,
+    const int tag,  /* unused */
+    const label comm
+)
+{
+    if
+    (
+        !UPstream::broadcast(const_cast<char*>(buf), bufSize, comm, rootProcNo)
+    )
+    {
+        FatalErrorInFunction
+            << "MPI_Bcast failure sending buffer data:" << label(bufSize) << nl
+            << Foam::abort(FatalError);
+        return false;
+    }
+
+    return true;
+}
+
+
+// ************************************************************************* //

src/Pstream/mpi/UOPstreamWrite.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2017 OpenFOAM Foundation
-    Copyright (C) 2019-2021 OpenCFD Ltd.
+    Copyright (C) 2019-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -24,9 +24,6 @@ License
     You should have received a copy of the GNU General Public License
     along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
 
-Description
-    Write primitive and binary block from OPstream
-
 \*---------------------------------------------------------------------------*/
 
 #include "UOPstream.H"
@@ -35,6 +32,22 @@ Description
 
 #include <mpi.h>
 
+// * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
+
+bool Foam::UOPstream::bufferIPCsend()
+{
+    return UOPstream::write
+    (
+        commsType(),
+        toProcNo_,
+        sendBuf_.cdata(),
+        sendBuf_.size(),
+        tag_,
+        comm_
+    );
+}
+
+
 // * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
 
 bool Foam::UOPstream::write
@@ -70,13 +83,13 @@ bool Foam::UOPstream::write
     PstreamGlobals::checkCommunicator(communicator, toProcNo);
 
 
-    bool transferFailed = true;
+    bool failed = true;
 
     profilingPstream::beginTiming();
 
     if (commsType == commsTypes::blocking)
     {
-        transferFailed = MPI_Bsend
+        failed = MPI_Bsend
         (
             const_cast<char*>(buf),
             bufSize,
@@ -99,7 +112,7 @@ bool Foam::UOPstream::write
     }
     else if (commsType == commsTypes::scheduled)
     {
-        transferFailed = MPI_Send
+        failed = MPI_Send
         (
             const_cast<char*>(buf),
             bufSize,
@@ -124,7 +137,7 @@ bool Foam::UOPstream::write
     {
         MPI_Request request;
 
-        transferFailed = MPI_Isend
+        failed = MPI_Isend
         (
             const_cast<char*>(buf),
             bufSize,
@@ -151,12 +164,11 @@ bool Foam::UOPstream::write
     else
     {
         FatalErrorInFunction
-            << "Unsupported communications type "
+            << "Unsupported communications type " << int(commsType)
-            << UPstream::commsTypeNames[commsType]
             << Foam::abort(FatalError);
     }
 
-    return !transferFailed;
+    return !failed;
 }
 
 

src/Pstream/mpi/UPstream.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2017 OpenFOAM Foundation
-    Copyright (C) 2016-2021 OpenCFD Ltd.
+    Copyright (C) 2016-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -40,17 +40,6 @@ License
 #include <cstdlib>
 #include <csignal>
 
-#if defined(WM_SP)
-    #define MPI_SCALAR MPI_FLOAT
-    #define MPI_SOLVESCALAR MPI_FLOAT
-#elif defined(WM_SPDP)
-    #define MPI_SCALAR MPI_FLOAT
-    #define MPI_SOLVESCALAR MPI_DOUBLE
-#elif defined(WM_DP)
-    #define MPI_SCALAR MPI_DOUBLE
-    #define MPI_SOLVESCALAR MPI_DOUBLE
-#endif
-
 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
 
 // The min value and default for MPI buffers length
@@ -491,245 +480,7 @@ void Foam::UPstream::abort()
 }
 
 
-void Foam::reduce
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-(
-    scalar& Value,
-    const sumOp<scalar>& bop,
-    const int tag,
-    const label communicator
-)
-{
-    if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
-    {
-        Pout<< "** reducing:" << Value << " with comm:" << communicator
-            << " warnComm:" << UPstream::warnComm
-            << endl;
-        error::printStack(Pout);
-    }
-    allReduce(Value, 1, MPI_SCALAR, MPI_SUM, bop, tag, communicator);
-}
-
-
-void Foam::reduce
-(
-    scalar& Value,
-    const minOp<scalar>& bop,
-    const int tag,
-    const label communicator
-)
-{
-    if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
-    {
-        Pout<< "** reducing:" << Value << " with comm:" << communicator
-            << " warnComm:" << UPstream::warnComm
-            << endl;
-        error::printStack(Pout);
-    }
-    allReduce(Value, 1, MPI_SCALAR, MPI_MIN, bop, tag, communicator);
-}
-
-
-void Foam::reduce
-(
-    vector2D& Value,
-    const sumOp<vector2D>& bop,
-    const int tag,
-    const label communicator
-)
-{
-    if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
-    {
-        Pout<< "** reducing:" << Value << " with comm:" << communicator
-            << " warnComm:" << UPstream::warnComm
-            << endl;
-        error::printStack(Pout);
-    }
-    allReduce(Value, 2, MPI_SCALAR, MPI_SUM, bop, tag, communicator);
-}
-
-
-void Foam::sumReduce
-(
-    scalar& Value,
-    label& Count,
-    const int tag,
-    const label communicator
-)
-{
-    if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
-    {
-        Pout<< "** sumReduce:" << Value << " with comm:" << communicator
-            << " warnComm:" << UPstream::warnComm
-            << endl;
-        error::printStack(Pout);
-    }
-    vector2D twoScalars(Value, scalar(Count));
-    reduce(twoScalars, sumOp<vector2D>(), tag, communicator);
-
-    Value = twoScalars.x();
-    Count = twoScalars.y();
-}
-
-
-void Foam::reduce
-(
-    scalar& Value,
-    const sumOp<scalar>& bop,
-    const int tag,
-    const label communicator,
-    label& requestID
-)
-{
-    iallReduce<scalar>(&Value, 1, MPI_SCALAR, MPI_SUM, communicator, requestID);
-}
-
-
-void Foam::reduce
-(
-    scalar values[],
-    const int size,
-    const sumOp<scalar>& bop,
-    const int tag,
-    const label communicator,
-    label& requestID
-)
-{
-    iallReduce<scalar>
-    (
-        values,
-        size,
-        MPI_SCALAR,
-        MPI_SUM,
-        communicator,
-        requestID
-    );
-}
-
-
-#if defined(WM_SPDP)
-void Foam::reduce
-(
-    solveScalar& Value,
-    const sumOp<solveScalar>& bop,
-    const int tag,
-    const label communicator
-)
-{
-    if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
-    {
-        Pout<< "** reducing:" << Value << " with comm:" << communicator
-            << " warnComm:" << UPstream::warnComm
-            << endl;
-        error::printStack(Pout);
-    }
-    allReduce(Value, 1, MPI_SOLVESCALAR, MPI_SUM, bop, tag, communicator);
-}
-
-
-void Foam::reduce
-(
-    solveScalar& Value,
-    const minOp<solveScalar>& bop,
-    const int tag,
-    const label communicator
-)
-{
-    if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
-    {
-        Pout<< "** reducing:" << Value << " with comm:" << communicator
-            << " warnComm:" << UPstream::warnComm
-            << endl;
-        error::printStack(Pout);
-    }
-    allReduce(Value, 1, MPI_SOLVESCALAR, MPI_MIN, bop, tag, communicator);
-}
-
-
-void Foam::reduce
-(
-    Vector2D<solveScalar>& Value,
-    const sumOp<Vector2D<solveScalar>>& bop,
-    const int tag,
-    const label communicator
-)
-{
-    if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
-    {
-        Pout<< "** reducing:" << Value << " with comm:" << communicator
-            << " warnComm:" << UPstream::warnComm
-            << endl;
-        error::printStack(Pout);
-    }
-    allReduce(Value, 2, MPI_SOLVESCALAR, MPI_SUM, bop, tag, communicator);
-}
-
-
-void Foam::sumReduce
-(
-    solveScalar& Value,
-    label& Count,
-    const int tag,
-    const label communicator
-)
-{
-    if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
-    {
-        Pout<< "** reducing:" << Value << " with comm:" << communicator
-            << " warnComm:" << UPstream::warnComm
-            << endl;
-        error::printStack(Pout);
-    }
-    Vector2D<solveScalar> twoScalars(Value, solveScalar(Count));
-    reduce(twoScalars, sumOp<Vector2D<solveScalar>>(), tag, communicator);
-
-    Value = twoScalars.x();
-    Count = twoScalars.y();
-}
-
-
-void Foam::reduce
-(
-    solveScalar& Value,
-    const sumOp<solveScalar>& bop,
-    const int tag,
-    const label communicator,
-    label& requestID
-)
-{
-    iallReduce<solveScalar>
-    (
-        &Value,
-        1,
-        MPI_SOLVESCALAR,
-        MPI_SUM,
-        communicator,
-        requestID
-    );
-}
-
-
-void Foam::reduce
-(
-    solveScalar values[],
-    const int size,
-    const sumOp<solveScalar>& bop,
-    const int tag,
-    const label communicator,
-    label& requestID
-)
-{
-    iallReduce<solveScalar>
-    (
-        values,
-        size,
-        MPI_SOLVESCALAR,
-        MPI_SUM,
-        communicator,
-        requestID
-    );
-}
-#endif
-
 
 void Foam::UPstream::allToAll
 (
@@ -1437,22 +1188,15 @@ int Foam::UPstream::allocateTag(const char* s)
 
     if (debug)
     {
-        //if (UPstream::lateBlocking > 0)
+        Pout<< "UPstream::allocateTag "
-        //{
+            << s << " : tag:" << tag << endl;
-        //    string& poutp = Pout.prefix();
-        //    poutp[poutp.size()-2*(UPstream::lateBlocking+2)+tag] = 'X';
-        //    Perr.prefix() = Pout.prefix();
-        //}
-        Pout<< "UPstream::allocateTag " << s
-            << " : tag:" << tag
-            << endl;
     }
 
     return tag;
 }
 
 
-int Foam::UPstream::allocateTag(const word& s)
+int Foam::UPstream::allocateTag(const std::string& s)
 {
     int tag;
     if (PstreamGlobals::freedTags_.size())
@@ -1466,15 +1210,8 @@ int Foam::UPstream::allocateTag(const word& s)
 
     if (debug)
     {
-        //if (UPstream::lateBlocking > 0)
+        Pout<< "UPstream::allocateTag "
-        //{
+            << s.c_str() << " : tag:" << tag << endl;
-        //    string& poutp = Pout.prefix();
-        //    poutp[poutp.size()-2*(UPstream::lateBlocking+2)+tag] = 'X';
-        //    Perr.prefix() = Pout.prefix();
-        //}
-        Pout<< "UPstream::allocateTag " << s
-            << " : tag:" << tag
-            << endl;
     }
 
     return tag;
@@ -1485,29 +1222,19 @@ void Foam::UPstream::freeTag(const char* s, const int tag)
 {
     if (debug)
     {
-        //if (UPstream::lateBlocking > 0)
+        Pout<< "UPstream::freeTag "
-        //{
+            << s << " tag:" << tag << endl;
-        //    string& poutp = Pout.prefix();
-        //    poutp[poutp.size()-2*(UPstream::lateBlocking+2)+tag] = ' ';
-        //    Perr.prefix() = Pout.prefix();
-        //}
-        Pout<< "UPstream::freeTag " << s << " tag:" << tag << endl;
     }
     PstreamGlobals::freedTags_.append(tag);
 }
 
 
-void Foam::UPstream::freeTag(const word& s, const int tag)
+void Foam::UPstream::freeTag(const std::string& s, const int tag)
 {
     if (debug)
     {
-        //if (UPstream::lateBlocking > 0)
+        Pout<< "UPstream::freeTag "
-        //{
+            << s.c_str() << " tag:" << tag << endl;
-        //    string& poutp = Pout.prefix();
-        //    poutp[poutp.size()-2*(UPstream::lateBlocking+2)+tag] = ' ';
-        //    Perr.prefix() = Pout.prefix();
-        //}
-        Pout<< "UPstream::freeTag " << s << " tag:" << tag << endl;
     }
     PstreamGlobals::freedTags_.append(tag);
 }

src/Pstream/mpi/UPstreamBroadcast.C

@@ -0,0 +1,86 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | www.openfoam.com
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+    Copyright (C) 2022 OpenCFD Ltd.
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "UPstream.H"
+#include "PstreamGlobals.H"
+#include "profilingPstream.H"
+
+#include <mpi.h>
+
+// * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
+
+bool Foam::UPstream::broadcast
+(
+    char* buf,
+    const std::streamsize bufSize,
+    const label communicator,
+    const int rootProcNo
+)
+{
+    if (!UPstream::parRun() || UPstream::nProcs(communicator) < 2)
+    {
+        // Nothing to do - ignore
+        return true;
+    }
+
+    //Needed?  PstreamGlobals::checkCommunicator(communicator, rootProcNo);
+
+    if (debug)
+    {
+        Pout<< "UPstream::broadcast : root:" << rootProcNo
+            << " comm:" << communicator
+            << " size:" << label(bufSize)
+            << Foam::endl;
+    }
+    if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
+    {
+        Pout<< "UPstream::broadcast : root:" << rootProcNo
+            << " comm:" << communicator
+            << " size:" << label(bufSize)
+            << " warnComm:" << UPstream::warnComm
+            << Foam::endl;
+        error::printStack(Pout);
+    }
+
+    profilingPstream::beginTiming();
+
+    bool failed = MPI_Bcast
+    (
+        buf,
+        bufSize,
+        MPI_BYTE,
+        rootProcNo,
+        PstreamGlobals::MPICommunicators_[communicator]
+    );
+
+    profilingPstream::addBroadcastTime();
+
+    return !failed;
+}
+
+
+// ************************************************************************* //

src/Pstream/mpi/UPstreamReduce.C

@@ -0,0 +1,211 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | www.openfoam.com
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+    Copyright (C) 2022 OpenCFD Ltd.
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "Pstream.H"
+#include "PstreamReduceOps.H"
+#include "allReduce.H"
+
+#include <mpi.h>
+#include <cinttypes>
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+// Specialisations for bool
+
+void Foam::reduce
+(
+    bool& value,
+    const andOp<bool>&,
+    const int tag,  /* (unused) */
+    const label comm
+)
+{
+    // This can also work:
+    // PstreamDetail::allReduce(&value, 1, MPI_BYTE, MPI_BAND, comm);
+    PstreamDetail::allReduce(&value, 1, MPI_C_BOOL, MPI_LAND, comm);
+}
+
+
+void Foam::reduce
+(
+    bool& value,
+    const orOp<bool>&,
+    const int tag,  /* (unused) */
+    const label comm
+)
+{
+    // This can also work:
+    // PstreamDetail::allReduce(&value, 1, MPI_BYTE, MPI_BOR, comm);
+    PstreamDetail::allReduce(&value, 1, MPI_C_BOOL, MPI_LOR, comm);
+}
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+// Specialisations for common reduction types
+
+#undef  Pstream_CommonReductions
+#define Pstream_CommonReductions(Native, TaggedType)                          \
+                                                                              \
+void Foam::reduce                                                             \
+(                                                                             \
+    Native& value,                                                            \
+    const minOp<Native>&,                                                     \
+    const int tag,  /* (unused) */                                            \
+    const label comm                                                          \
+)                                                                             \
+{                                                                             \
+    PstreamDetail::allReduce<Native>                                          \
+    (                                                                         \
+        &value, 1, TaggedType, MPI_MIN, comm                                  \
+    );                                                                        \
+}                                                                             \
+                                                                              \
+void Foam::reduce                                                             \
+(                                                                             \
+    Native& value,                                                            \
+    const maxOp<Native>&,                                                     \
+    const int tag,  /* (unused) */                                            \
+    const label comm                                                          \
+)                                                                             \
+{                                                                             \
+    PstreamDetail::allReduce<Native>                                          \
+    (                                                                         \
+        &value, 1, TaggedType, MPI_MAX, comm                                  \
+    );                                                                        \
+}                                                                             \
+                                                                              \
+void Foam::reduce                                                             \
+(                                                                             \
+    Native& value,                                                            \
+    const sumOp<Native>&,                                                     \
+    const int tag,  /* (unused) */                                            \
+    const label comm                                                          \
+)                                                                             \
+{                                                                             \
+    PstreamDetail::allReduce<Native>                                          \
+    (                                                                         \
+        &value, 1, TaggedType, MPI_SUM, comm                                  \
+    );                                                                        \
+}                                                                             \
+                                                                              \
+void Foam::reduce                                                             \
+(                                                                             \
+    Native values[],                                                          \
+    const int size,                                                           \
+    const sumOp<Native>&,                                                     \
+    const int tag,  /* (unused) */                                            \
+    const label comm                                                          \
+)                                                                             \
+{                                                                             \
+    PstreamDetail::allReduce<Native>                                          \
+    (                                                                         \
+        values, size, TaggedType, MPI_SUM, comm                               \
+    );                                                                        \
+}                                                                             \
+
+
+Pstream_CommonReductions(int32_t, MPI_INT32_T);
+Pstream_CommonReductions(int64_t, MPI_INT64_T);
+Pstream_CommonReductions(uint32_t, MPI_UINT32_T);
+Pstream_CommonReductions(uint64_t, MPI_UINT64_T);
+Pstream_CommonReductions(float,   MPI_FLOAT);
+Pstream_CommonReductions(double,  MPI_DOUBLE);
+
+#undef Pstream_CommonReductions
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+// Specialisations for floating-point types
+
+#undef  Pstream_FloatReductions
+#define Pstream_FloatReductions(Native, TaggedType)                           \
+                                                                              \
+void Foam::sumReduce                                                          \
+(                                                                             \
+    Native& value,                                                            \
+    label& count,                                                             \
+    const int tag,  /* (unused) */                                            \
+    const label comm                                                          \
+)                                                                             \
+{                                                                             \
+    if (UPstream::parRun())                                                   \
+    {                                                                         \
+        Native values[2];                                                     \
+        values[0] = value;                                                    \
+        values[1] = static_cast<Native>(count);                               \
+                                                                              \
+        PstreamDetail::allReduce<Native>                                      \
+        (                                                                     \
+            values, 2, TaggedType, MPI_SUM, comm                              \
+        );                                                                    \
+                                                                              \
+        value = values[0];                                                    \
+        count = static_cast<label>(values[1]);                                \
+    }                                                                         \
+}                                                                             \
+                                                                              \
+void Foam::reduce                                                             \
+(                                                                             \
+    Native& value,                                                            \
+    const sumOp<Native>&,                                                     \
+    const int tag,  /* (unused) */                                            \
+    const label comm,                                                         \
+    label& requestID                                                          \
+)                                                                             \
+{                                                                             \
+    PstreamDetail::iallReduce<Native>                                         \
+    (                                                                         \
+        &value, 1, TaggedType, MPI_SUM, comm, requestID                       \
+    );                                                                        \
+}                                                                             \
+                                                                              \
+void Foam::reduce                                                             \
+(                                                                             \
+    Native values[],                                                          \
+    const int size,                                                           \
+    const sumOp<Native>&,                                                     \
+    const int tag,  /* (unused) */                                            \
+    const label comm,                                                         \
+    label& requestID                                                          \
+)                                                                             \
+{                                                                             \
+    PstreamDetail::iallReduce<Native>                                         \
+    (                                                                         \
+        values, size, TaggedType, MPI_SUM, comm, requestID                    \
+    );                                                                        \
+}
+
+
+Pstream_FloatReductions(float, MPI_FLOAT);
+Pstream_FloatReductions(double, MPI_DOUBLE);
+
+#undef Pstream_FloatReductions
+
+
+// ************************************************************************* //

src/Pstream/mpi/allReduce.H

@@ -6,6 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2012-2016 OpenFOAM Foundation
+    Copyright (C) 2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -23,54 +24,69 @@ License
     You should have received a copy of the GNU General Public License
     along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
 
-InNamespace
+Namespace
-    Foam
+    Foam::PstreamDetail
 
 Description
-    Various functions to wrap MPI_Allreduce
+    Some implementation details for Pstream and/or MPI.
+
+InNamespace
+    Foam::PstreamDetail
+
+Description
+    Functions to wrap MPI_Bcast, MPI_Allreduce, MPI_Iallreduce
 
 SourceFiles
     allReduceTemplates.C
 
 \*---------------------------------------------------------------------------*/
 
-#ifndef allReduce_H
+#ifndef Foam_allReduce_H
-#define allReduce_H
+#define Foam_allReduce_H
 
 #include "UPstream.H"
-
 #include <mpi.h>
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
 namespace Foam
 {
+namespace PstreamDetail
+{
 
-template<class Type, class BinaryOp>
+template<class Type>
+void allBroadcast
+(
+    Type* values,
+    int count,
+    MPI_Datatype datatype,
+    const label communicator
+);
+
+template<class Type>
 void allReduce
 (
-    Type& Value,
+    Type* values,
     int count,
-    MPI_Datatype MPIType,
+    MPI_Datatype datatype,
-    MPI_Op op,
+    MPI_Op optype,
-    const BinaryOp& bop,
-    const int tag,
     const label communicator
 );
 
 template<class Type>
 void iallReduce
 (
-    void* Value,
+    Type* values,
     int count,
-    MPI_Datatype MPIType,
+    MPI_Datatype datatype,
-    MPI_Op op,
+    MPI_Op optype,
     const label communicator,
     label& requestID
 );
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
+} // End namespace PstreamDetail
 } // End namespace Foam
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //

src/Pstream/mpi/allReduceTemplates.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2012-2015 OpenFOAM Foundation
-    Copyright (C) 2019-2020 OpenCFD Ltd.
+    Copyright (C) 2019-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -28,18 +28,16 @@ License
 
 #include "allReduce.H"
 #include "profilingPstream.H"
+#include "PstreamGlobals.H"
 
 // * * * * * * * * * * * * * * * Global Functions  * * * * * * * * * * * * * //
 
-template<class Type, class BinaryOp>
+template<class Type>
-void Foam::allReduce
+void Foam::PstreamDetail::allBroadcast
 (
-    Type& Value,
+    Type* values,
-    int MPICount,
+    int count,
-    MPI_Datatype MPIType,
+    MPI_Datatype datatype,
-    MPI_Op MPIOp,
-    const BinaryOp& bop,
-    const int tag,
     const label communicator
 )
 {
@@ -50,129 +48,75 @@ void Foam::allReduce
 
     profilingPstream::beginTiming();
 
-    if (UPstream::nProcs(communicator) <= UPstream::nProcsSimpleSum)
+    // const int retval =
-    {
+    MPI_Bcast
-        if (UPstream::master(communicator))
-        {
-            for (const int proci : UPstream::subProcs(communicator))
-            {
-                Type value;
-
-                if
     (
-                    MPI_Recv
+        values,
-                    (
+        count,
-                        &value,
+        datatype,
-                        MPICount,
+        0,  // (root process) is master == UPstream::masterNo()
-                        MPIType,
-                        proci,
-                        tag,
-                        PstreamGlobals::MPICommunicators_[communicator],
-                        MPI_STATUS_IGNORE
-                    )
-                )
-                {
-                    FatalErrorInFunction
-                        << "MPI_Recv failed"
-                        << Foam::abort(FatalError);
-                }
-
-                Value = bop(Value, value);
-            }
-        }
-        else
-        {
-            if
-            (
-                MPI_Send
-                (
-                    &Value,
-                    MPICount,
-                    MPIType,
-                    UPstream::masterNo(),
-                    tag,
-                    PstreamGlobals::MPICommunicators_[communicator]
-                )
-            )
-            {
-                FatalErrorInFunction
-                    << "MPI_Send failed"
-                    << Foam::abort(FatalError);
-            }
-        }
-
-
-        if (UPstream::master(communicator))
-        {
-            for (const int proci : UPstream::subProcs(communicator))
-            {
-                if
-                (
-                    MPI_Send
-                    (
-                        &Value,
-                        MPICount,
-                        MPIType,
-                        proci,
-                        tag,
-                        PstreamGlobals::MPICommunicators_[communicator]
-                    )
-                )
-                {
-                    FatalErrorInFunction
-                        << "MPI_Send failed"
-                        << Foam::abort(FatalError);
-                }
-            }
-        }
-        else
-        {
-            if
-            (
-                MPI_Recv
-                (
-                    &Value,
-                    MPICount,
-                    MPIType,
-                    UPstream::masterNo(),
-                    tag,
-                    PstreamGlobals::MPICommunicators_[communicator],
-                    MPI_STATUS_IGNORE
-                )
-            )
-            {
-                FatalErrorInFunction
-                    << "MPI_Recv failed"
-                    << Foam::abort(FatalError);
-            }
-        }
-    }
-    else
-    {
-        Type sum;
-        MPI_Allreduce
-        (
-            &Value,
-            &sum,
-            MPICount,
-            MPIType,
-            MPIOp,
         PstreamGlobals::MPICommunicators_[communicator]
     );
-        Value = sum;
+
+    profilingPstream::addBroadcastTime();
 }
 
+
+template<class Type>
+void Foam::PstreamDetail::allReduce
+(
+    Type* values,
+    int count,
+    MPI_Datatype datatype,
+    MPI_Op optype,
+    const label communicator
+)
+{
+    if (!UPstream::parRun())
+    {
+        return;
+    }
+
+    if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
+    {
+        Pout<< "** reducing:";
+        if (count == 1)
+        {
+            Pout<< (*values);
+        }
+        else
+        {
+            Pout<< UList<Type>(values, count);
+        }
+        Pout<< " with comm:" << communicator
+            << " warnComm:" << UPstream::warnComm << endl;
+        error::printStack(Pout);
+    }
+
+    profilingPstream::beginTiming();
+
+    // const int retval =
+    MPI_Allreduce
+    (
+        MPI_IN_PLACE,
+        values,
+        count,
+        datatype,
+        optype,
+        PstreamGlobals::MPICommunicators_[communicator]
+    );
+
     profilingPstream::addReduceTime();
 }
 
 
 template<class Type>
-void Foam::iallReduce
+void Foam::PstreamDetail::iallReduce
 (
-    void* recvBuf,
+    Type* values,
-    int MPICount,
+    int count,
-    MPI_Datatype MPIType,
+    MPI_Datatype datatype,
-    MPI_Op MPIOp,
+    MPI_Op optype,
     const label communicator,
     label& requestID
 )
@@ -184,9 +128,16 @@ void Foam::iallReduce
 
     if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
     {
-        Pout<< "** non-blocking reducing:"
+        Pout<< "** non-blocking reducing:";
-            << UList<Type>(static_cast<Type*>(recvBuf), MPICount)
+        if (count == 1)
-            << " with comm:" << communicator
+        {
+            Pout<< (*values);
+        }
+        else
+        {
+            Pout<< UList<Type>(values, count);
+        }
+        Pout<< " with comm:" << communicator
             << " warnComm:" << UPstream::warnComm << endl;
         error::printStack(Pout);
     }
@@ -200,10 +151,10 @@ void Foam::iallReduce
         MPI_Iallreduce
         (
             MPI_IN_PLACE,
-            recvBuf,
+            values,
-            MPICount,
+            count,
-            MPIType,
+            datatype,
-            MPIOp,
+            optype,
             PstreamGlobals::MPICommunicators_[communicator],
             &request
         )
@@ -211,7 +162,7 @@ void Foam::iallReduce
     {
         FatalErrorInFunction
             << "MPI_Iallreduce failed for "
-            << UList<Type>(static_cast<Type*>(recvBuf), MPICount)
+            << UList<Type>(values, count)
             << Foam::abort(FatalError);
     }
 
@@ -238,17 +189,17 @@ void Foam::iallReduce
         MPI_Allreduce
         (
             MPI_IN_PLACE,
-            recvBuf,
+            values,
-            MPICount,
+            count,
-            MPIType,
+            datatype,
-            MPIOp,
+            optype,
             PstreamGlobals::MPICommunicators_[communicator]
         )
     )
     {
         FatalErrorInFunction
             << "MPI_Allreduce failed for "
-            << UList<Type>(static_cast<Type*>(recvBuf), MPICount)
+            << UList<Type>(values, count)
             << Foam::abort(FatalError);
     }
     requestID = -1;

src/dynamicMesh/fvMeshTools/fvMeshTools.C

@@ -478,25 +478,10 @@ Foam::autoPtr<Foam::fvMesh> Foam::fvMeshTools::newMesh
             )
         );
         Pstream::parRun(oldParRun);
-
-        // Send patches
-        for (const int slave : Pstream::subProcs())
-        {
-            OPstream toSlave(Pstream::commsTypes::scheduled, slave);
-            toSlave << patchEntries;
-        }
-    }
-    else
-    {
-        // Receive patches
-        IPstream fromMaster
-        (
-            Pstream::commsTypes::scheduled,
-            Pstream::masterNo()
-        );
-        fromMaster >> patchEntries;
     }
 
+    // Broadcast information to all
+    Pstream::scatter(patchEntries);
     Pstream::scatter(facesInstance);
     Pstream::scatter(pointsInstance);
 

src/finiteArea/faMesh/faPatches/constraint/processor/processorFaPatch.C

@@ -28,8 +28,6 @@ License
 
 #include "processorFaPatch.H"
 #include "addToRunTimeSelectionTable.H"
-#include "IPstream.H"
-#include "OPstream.H"
 #include "transformField.H"
 #include "faBoundaryMesh.H"
 #include "faMesh.H"

src/finiteArea/fields/faPatchFields/constraint/processor/processorFaPatchField.C

@@ -28,8 +28,6 @@ License
 
 #include "processorFaPatchField.H"
 #include "processorFaPatch.H"
-#include "IPstream.H"
-#include "OPstream.H"
 #include "transformField.H"
 
 // * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //

src/finiteVolume/fvMesh/zoneDistribute/zoneDistribute.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2020 DLR
-    Copyright (C) 2020 OpenCFD Ltd.
+    Copyright (C) 2020-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -27,13 +27,6 @@ License
 \*---------------------------------------------------------------------------*/
 
 #include "zoneDistribute.H"
-#include "dummyTransform.H"
-#include "emptyPolyPatch.H"
-#include "processorPolyPatch.H"
-#include "syncTools.H"
-#include "wedgePolyPatch.H"
-
-#include "globalPoints.H"
 
 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
 
@@ -43,61 +36,15 @@ namespace Foam
 }
 
 
-// * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
-
-Foam::autoPtr<Foam::indirectPrimitivePatch>
-Foam::zoneDistribute::coupledFacesPatch() const
-{
-    const polyBoundaryMesh& patches = mesh_.boundaryMesh();
-
-    label nCoupled = 0;
-
-    for (const polyPatch& pp : patches)
-    {
-        if (isA<processorPolyPatch>(pp))
-        {
-            nCoupled += pp.size();
-        }
-    }
-    labelList coupledFaces(nCoupled);
-    nCoupled = 0;
-
-    for (const polyPatch& pp : patches)
-    {
-        if (isA<processorPolyPatch>(pp))
-        {
-            label facei = pp.start();
-
-            forAll(pp, i)
-            {
-                coupledFaces[nCoupled++] = facei++;
-            }
-        }
-    }
-
-    return autoPtr<indirectPrimitivePatch>::New
-    (
-        IndirectList<face>
-        (
-            mesh_.faces(),
-            coupledFaces
-        ),
-        mesh_.points()
-    );
-}
-
-
 // * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
 
 Foam::zoneDistribute::zoneDistribute(const fvMesh& mesh)
 :
     MeshObject<fvMesh, Foam::TopologicalMeshObject, zoneDistribute>(mesh),
-    coupledBoundaryPoints_(coupledFacesPatch()().meshPoints()),
-    send_(Pstream::nProcs()),
     stencil_(zoneCPCStencil::New(mesh)),
-    gblIdx_(stencil_.globalNumbering())
+    globalNumbering_(stencil_.globalNumbering()),
-{
+    send_(UPstream::nProcs())
-}
+{}
 
 
 // * * * * * * * * * * * * * * * * Selectors  * * * * * * * * * * * * * * //
@@ -124,7 +71,11 @@ void Foam::zoneDistribute::updateStencil(const boolList& zone)
 }
 
 
-void Foam::zoneDistribute::setUpCommforZone(const boolList& zone,bool updateStencil)
+void Foam::zoneDistribute::setUpCommforZone
+(
+    const boolList& zone,
+    bool updateStencil
+)
 {
     zoneCPCStencil& stencil = zoneCPCStencil::New(mesh_);
 
@@ -133,60 +84,56 @@ void Foam::zoneDistribute::setUpCommforZone(const boolList& zone,bool updateSten
         stencil.updateStencil(zone);
     }
 
-    const labelHashSet comms = stencil.needsComm();
+    if (UPstream::parRun())
-
-    List<labelHashSet> needed(Pstream::nProcs());
-
-    if (Pstream::parRun())
     {
-        for (const label celli : comms)
+        List<labelHashSet> needed(UPstream::nProcs());
+
+        // Bin according to originating (sending) processor
+        for (const label celli : stencil.needsComm())
         {
             if (zone[celli])
             {
                 for (const label gblIdx : stencil_[celli])
                 {
-                    if (!gblIdx_.isLocal(gblIdx))
+                    if (!globalNumbering_.isLocal(gblIdx))
                     {
-                        const label procID = gblIdx_.whichProcID (gblIdx);
+                        const label procID =
+                            globalNumbering_.whichProcID(gblIdx);
                         needed[procID].insert(gblIdx);
                     }
                 }
             }
         }
 
-        PstreamBuffers pBufs(Pstream::commsTypes::nonBlocking);
+        // Stream the send data into PstreamBuffers,
+        // which we also use to track the current topology.
 
-        // Stream data into buffer
+        PstreamBuffers pBufs(UPstream::commsTypes::nonBlocking);
-        for (const int domain : Pstream::allProcs())
+
+        for (const int proci : UPstream::allProcs())
         {
-            if (domain != Pstream::myProcNo())
+            if (proci != UPstream::myProcNo() && !needed[proci].empty())
             {
-                // Put data into send buffer
+                // Serialize as List
-                UOPstream toDomain(domain, pBufs);
+                UOPstream toProc(proci, pBufs);
-
+                toProc << needed[proci].sortedToc();
-                toDomain << needed[domain];
             }
         }
 
-        // wait until everything is written.
+        pBufs.finishedSends(sendConnections_, sendProcs_, recvProcs_);
-        pBufs.finishedSends();
 
-        for (const int domain : Pstream::allProcs())
+        for (const int proci : pBufs.allProcs())
         {
-            send_[domain].clear();
+            send_[proci].clear();
 
-            if (domain != Pstream::myProcNo())
+            if (proci != UPstream::myProcNo() && pBufs.hasRecvData(proci))
             {
-                // get data from send buffer
+                UIPstream fromProc(proci, pBufs);
-                UIPstream fromDomain(domain, pBufs);
+                fromProc >> send_[proci];
-
-                fromDomain >> send_[domain];
             }
         }
     }
 }
 
 
-
-
 // ************************************************************************* //

src/finiteVolume/fvMesh/zoneDistribute/zoneDistribute.H

@@ -6,6 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2020 DLR
+    Copyright (C) 2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -30,29 +31,44 @@ Description
     Class for parallel communication in a narrow band. It either provides a Map
     with the neighbouring values of the selected region or returns a Map of the
     required values in global addressing. Also holds a reference to the stencil
-    Before the data transfer the communation has to be set up:
+    Before the data transfer the communication has to be set up:
     exchangeFields_.setUpCommforZone(interfaceCell_);
     Is used in the plicRDF
 
     Original code supplied by Henning Scheufler, DLR (2019)
 
+    Additional optimization of processor communication
+    provided by Tetsuo AOYAGI, RIST (2022), to use a more compact
+    exchange of sizes with an updated version of PstreamBuffers.
+    This optimization uses additional sendTo/recvFrom member data
+    to track the topological connectivity, acting like an on-the-fly
+    sub-communicator, and respects corner connectivity.
+
+    -# Initially topological connections are empty (or all false).
+    -# Scan the stencil global cellIds (active zones only) and split
+       into sub-lists according the originating processor (the sender).
+    -# If an originating processor appears/disappears, need to update
+       the connectivity information (requires an all-to-all).
+    -# When possible, the topological send/recv is used in PstreamBuffers
+       finishedSends (minimizes communication).
+    .
+
 SourceFiles
+    zoneDistributeI.H
     zoneDistribute.C
 
 \*---------------------------------------------------------------------------*/
 
-#ifndef zoneDistribute_H
+#ifndef Foam_zoneDistribute_H
-#define zoneDistribute_H
+#define Foam_zoneDistribute_H
 
 #include "fvMesh.H"
 #include "globalIndex.H"
 #include "volFields.H"
 
 #include "zoneCPCStencil.H"
-#include "IOobject.H"
 #include "MeshObject.H"
 
-
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
 namespace Foam
@@ -68,37 +84,41 @@ class zoneDistribute
 {
     // Private Data
 
-
+        //- Reference to the zone stencil
-        //- labels of the points on coupled patches
-        labelList coupledBoundaryPoints_;
-
-        //- storage of the addressing for processor-to-processor comms
-        List<labelHashSet> send_;
-
-        //- Return patch of all coupled faces.
-        autoPtr<indirectPrimitivePatch> coupledFacesPatch() const;
-
         zoneCPCStencil& stencil_;
 
-        const globalIndex& gblIdx_;
+        //- Global number into index of cells/faces
+        const globalIndex& globalNumbering_;
+
+        //- Global cell/face index to send for processor-to-processor comms
+        List<labelList> send_;
+
+        //- Parallel [cache]: send connectivity (true/false)
+        bitSet sendConnections_;
+
+        //- Parallel [cache]: send data to these ranks
+        DynamicList<label> sendProcs_;
+
+        //- Parallel [cache]: recv data from these ranks
+        DynamicList<label> recvProcs_;
 
 
-        //- Gives patchNumber and patchFaceNumber for a given
+    // Private Member Functions
-        //- Geometric volume field
+
+        //- Return local volField value at (cell or face) index
         template<typename Type>
         Type getLocalValue
         (
-            const GeometricField<Type, fvPatchField, volMesh>& phi,
+            const VolumeField<Type>& phi,
             const label localIdx
         ) const;
 
-
         //- Gives patchNumber and patchFaceNumber for a given
         //- Geometric volume field
         template<typename Type>
         Type faceValue
         (
-            const GeometricField<Type, fvPatchField, volMesh>& phi,
+            const VolumeField<Type>& phi,
             const label localIdx
         ) const;
 
@@ -114,14 +134,11 @@ public:
         //- Construct from fvMesh
         explicit zoneDistribute(const fvMesh&);
 
-
+        //- Selector
-    // Selectors
-
         static zoneDistribute& New(const fvMesh&);
 
 
     //- Destructor
-
     virtual ~zoneDistribute() = default;
 
 
@@ -142,7 +159,7 @@ public:
         //- Addressing reference
         const globalIndex& globalNumbering() const noexcept
         {
-            return gblIdx_;
+            return globalNumbering_;
         }
 
         //- Gives patchNumber and patchFaceNumber for a given
@@ -150,7 +167,7 @@ public:
         template<typename Type>
         Type getValue
         (
-            const GeometricField<Type, fvPatchField, volMesh>& phi,
+            const VolumeField<Type>& phi,
             const Map<Type>& valuesFromOtherProc,
             const label gblIdx
         ) const;
@@ -160,7 +177,7 @@ public:
         Map<Field<Type>> getFields
         (
             const boolList& zone,
-            const GeometricField<Type, fvPatchField, volMesh>& phi
+            const VolumeField<Type>& phi
         );
 
         //- Returns stencil and provides a Map with globalNumbering
@@ -168,10 +185,8 @@ public:
         Map<Type> getDatafromOtherProc
         (
             const boolList& zone,
-            const GeometricField<Type, fvPatchField, volMesh>& phi
+            const VolumeField<Type>& phi
         );
-
-
 };
 
 

src/finiteVolume/fvMesh/zoneDistribute/zoneDistributeI.H

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2019-2020 DLR
-    Copyright (C) 2020 OpenCFD Ltd.
+    Copyright (C) 2020-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -26,16 +26,14 @@ License
 
 \*---------------------------------------------------------------------------*/
 
-#include "zoneDistribute.H"
 #include "DynamicField.H"
-#include "syncTools.H"
 
 // * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
 
 template<typename Type>
 Type Foam::zoneDistribute::getLocalValue
 (
-    const GeometricField<Type, fvPatchField, volMesh>& phi,
+    const VolumeField<Type>& phi,
     const label localIdx
 ) const
 {
@@ -51,7 +49,7 @@ Type Foam::zoneDistribute::getLocalValue
 template<typename Type>
 Type Foam::zoneDistribute::faceValue
 (
-    const GeometricField<Type, fvPatchField, volMesh>& phi,
+    const VolumeField<Type>& phi,
     const label localIdx
 ) const
 {
@@ -80,18 +78,19 @@ Type Foam::zoneDistribute::faceValue
 template<typename Type>
 Type Foam::zoneDistribute::getValue
 (
-    const GeometricField<Type, fvPatchField, volMesh>& phi,
+    const VolumeField<Type>& phi,
     const Map<Type>& valuesFromOtherProc,
     const label gblIdx
 ) const
 {
-    if (gblIdx_.isLocal(gblIdx))
+    if (globalNumbering_.isLocal(gblIdx))
     {
-        const label idx = gblIdx_.toLocal(gblIdx);
+        const label localIdx = globalNumbering_.toLocal(gblIdx);
-        return getLocalValue(phi,idx);
+        return getLocalValue(phi,localIdx);
     }
-    else // from other proc
+    else
     {
+        // From other proc
         return valuesFromOtherProc[gblIdx];
     }
 }
@@ -101,7 +100,7 @@ template<typename Type>
 Foam::Map<Foam::Field<Type>> Foam::zoneDistribute::getFields
 (
     const boolList& zone,
-    const GeometricField<Type, fvPatchField, volMesh>& phi
+    const VolumeField<Type>& phi
 )
 {
     if (zone.size() != phi.size())
@@ -111,7 +110,6 @@ Foam::Map<Foam::Field<Type>> Foam::zoneDistribute::getFields
             << "size of phi:" <<  phi.size()
             << "do not match. Did the mesh change?"
             << exit(FatalError);
-
     }
 
 
@@ -120,20 +118,20 @@ Foam::Map<Foam::Field<Type>> Foam::zoneDistribute::getFields
 
     Map<Field<Type>> stencilWithValues;
 
-    DynamicField<Type> tmpField(100);
+    DynamicField<Type> tmpField(128);
 
     forAll(zone, celli)
     {
         if (zone[celli])
         {
-            tmpField.clearStorage();
+            tmpField.clear();
 
             for (const label gblIdx : stencil_[celli])
             {
                 tmpField.append(getValue(phi,neiValues,gblIdx));
             }
 
-            stencilWithValues.insert(celli,tmpField);
+            stencilWithValues.emplace(celli, tmpField);
         }
     }
 
@@ -145,7 +143,7 @@ template<typename Type>
 Foam::Map<Type> Foam::zoneDistribute::getDatafromOtherProc
 (
     const boolList& zone,
-    const GeometricField<Type, fvPatchField, volMesh>& phi
+    const VolumeField<Type>& phi
 )
 {
     if (zone.size() != phi.size())
@@ -155,57 +153,59 @@ Foam::Map<Type> Foam::zoneDistribute::getDatafromOtherProc
             << "size of phi:" <<  phi.size()
             << "do not match. Did the mesh change?"
             << exit(FatalError);
-
     }
 
 
     // Get values from other proc
     Map<Type> neiValues;
-    List<Map<Type>> sendValues(Pstream::nProcs());
 
-    if (Pstream::parRun())
+    if (UPstream::parRun())
     {
-        forAll(send_, domaini)
+        if (sendConnections_.empty())
         {
-            for (const label sendIdx : send_[domaini])
+            WarningInFunction
+                << "The send/recv connections not initialized - "
+                << "likely that setUpCommforZone() was not called"
+                << endl;
+            // But don't exit/abort for now
+        }
+
+        // Stream the send data into PstreamBuffers,
+        // which we also use to track the current topology.
+
+        PstreamBuffers pBufs(UPstream::commsTypes::nonBlocking);
+
+        for (const int proci : UPstream::allProcs())
         {
-                sendValues[domaini].insert
+            if (proci != UPstream::myProcNo() && !send_[proci].empty())
+            {
+                // Serialize as Map
+                Map<Type> sendValues(2*send_[proci].size());
+
+                for (const label sendIdx : send_[proci])
+                {
+                    sendValues.insert
                     (
                         sendIdx,
-                    getLocalValue(phi,gblIdx_.toLocal(sendIdx))
+                        getLocalValue(phi, globalNumbering_.toLocal(sendIdx))
                     );
                 }
-        }
 
-        PstreamBuffers pBufs(Pstream::commsTypes::nonBlocking);
+                UOPstream toProc(proci, pBufs);
-
+                toProc << sendValues;
-        // Stream data into buffer
-        for (const int domain : Pstream::allProcs())
-        {
-            if (domain != Pstream::myProcNo())
-            {
-                // Put data into send buffer
-                UOPstream toDomain(domain, pBufs);
-
-                toDomain << sendValues[domain];
             }
         }
 
-        // Wait until everything is written.
+        pBufs.finishedSends(sendConnections_, sendProcs_, recvProcs_);
-        pBufs.finishedSends();
 
-        Map<Type> tmpValue;
+        for (const int proci : pBufs.allProcs())
-
-        for (const int domain : Pstream::allProcs())
         {
-            if (domain != Pstream::myProcNo())
+            if (proci != UPstream::myProcNo() && pBufs.hasRecvData(proci))
             {
-                // Get data from send buffer
+                UIPstream fromProc(proci, pBufs);
-                UIPstream fromDomain(domain, pBufs);
+                Map<Type> tmpValues(fromProc);
 
-                fromDomain >> tmpValue;
+                neiValues += tmpValues;
-
-                neiValues += tmpValue;
             }
         }
     }

src/functionObjects/field/expressions/fvExpressionField.C

@@ -265,8 +265,8 @@ bool Foam::functionObjects::fvExpressionField::setField
 
     if (action_ == actionType::opModify && log)
     {
-        const label numTotal = returnReduce(output.size(), plusOp<label>());
+        const label numTotal = returnReduce(output.size(), sumOp<label>());
-        reduce(numValuesChanged, plusOp<label>());
+        reduce(numValuesChanged, sumOp<label>());
 
         Info<< this->name() << ": set ";
         if (numValuesChanged == numTotal)

src/functionObjects/randomProcesses/energySpectrum/energySpectrum.C

@@ -223,7 +223,7 @@ bool Foam::functionObjects::energySpectrum::write()
     {
         PstreamBuffers pBufs(Pstream::commsTypes::nonBlocking);
 
-        UOPstream toProc(0, pBufs);
+        UOPstream toProc(Pstream::masterNo(), pBufs);
 
         toProc << Uc << C << cellAddr_;
 

src/functionObjects/utilities/parProfiling/parProfiling.C

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2019 OpenCFD Ltd.
+    Copyright (C) 2019-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -50,45 +50,6 @@ namespace functionObjects
     );
 
 } // End namespace functionObject
-
-
-    // Processor and time for each of: -min -max -sum
-    typedef FixedList<Tuple2<label, scalar>, 3> statData;
-
-
-    //- Reduction class. If x and y are not equal assign value.
-    struct statsEqOp
-    {
-        void operator()
-        (
-            FixedList<statData, 2>& xStats,
-            const FixedList<statData, 2>& yStats
-        ) const
-        {
-            forAll(xStats, i)
-            {
-                statData& x = xStats[i];
-                const statData& y = yStats[i];
-
-                // 0 : min
-                // 1 : max
-                // 2 : sum
-                if (x[0].second() > y[0].second())
-                {
-                    x[0].second() = y[0].second();
-                    x[0].first()  = y[0].first();
-                }
-                if (x[1].second() < y[1].second())
-                {
-                    x[1].second() = y[1].second();
-                    x[1].first()  = y[1].first();
-                }
-                x[2].second() += y[2].second();
-                x[2].first()++;
-            }
-        }
-    };
-
 } // End namespace Foam
 
 
@@ -124,57 +85,59 @@ void Foam::functionObjects::parProfiling::report()
         return;
     }
 
-    typedef FixedList<Tuple2<label, scalar>, 3> statData;
+    // (Time, Processor) for each of: min/max/sum
-    FixedList<statData, 2> times;
+    typedef FixedList<Tuple2<double, int>, 3> statData;
+    typedef FixedList<statData, 2> statDataTimes;
+
+    // Reduction: if x and y are unequal assign value.
+    auto statsEqOp = [](statDataTimes& xStats, const statDataTimes& yStats)
+    {
+        forAll(xStats, i)
+        {
+            statData& x = xStats[i];
+            const statData& y = yStats[i];
+
+            // 0: min, 1: max, 2: total (or avg)
+            if (x[0].first() > y[0].first())
+            {
+                x[0] = y[0];
+            }
+            if (x[1].first() < y[1].first())
+            {
+                x[1] = y[1];
+            }
+            x[2].first() += y[2].first();
+        }
+    };
+
+    statDataTimes times;
 
     {
-        const scalar masterTime =
+        const double masterTime =
         (
             profilingPstream::times(profilingPstream::REDUCE)
           + profilingPstream::times(profilingPstream::GATHER)
           + profilingPstream::times(profilingPstream::SCATTER)
+            // Include broadcast with reduce instead of all-to-all
+          + profilingPstream::times(profilingPstream::BROADCAST)
         );
 
-        statData& reduceStats = times[0];
+        times[0] = Tuple2<double, int>(masterTime, Pstream::myProcNo());
-
-        Tuple2<label, scalar>& minTime = reduceStats[0];
-        minTime.first() = Pstream::myProcNo();
-        minTime.second() = masterTime;
-
-        Tuple2<label, scalar>& maxTime = reduceStats[1];
-        maxTime.first() = Pstream::myProcNo();
-        maxTime.second() = masterTime;
-
-        Tuple2<label, scalar>& sumTime = reduceStats[2];
-        sumTime.first() = 1;
-        sumTime.second() = masterTime;
     }
 
     {
-        const scalar allTime =
+        const double allTime =
         (
             profilingPstream::times(profilingPstream::WAIT)
           + profilingPstream::times(profilingPstream::ALL_TO_ALL)
         );
 
-        statData& allToAllStats = times[1];
+        times[1] = Tuple2<double, int>(allTime, Pstream::myProcNo());
-
-        Tuple2<label, scalar>& minTime = allToAllStats[0];
-        minTime.first() = Pstream::myProcNo();
-        minTime.second() = allTime;
-
-        Tuple2<label, scalar>& maxTime = allToAllStats[1];
-        maxTime.first() = Pstream::myProcNo();
-        maxTime.second() = allTime;
-
-        Tuple2<label, scalar>& sumTime = allToAllStats[2];
-        sumTime.first() = 1;
-        sumTime.second() = allTime;
     }
 
     profilingPstream::suspend();
 
-    Pstream::combineGather(times, statsEqOp());
+    Pstream::combineGather(times, statsEqOp);
 
     profilingPstream::resume();
 
@@ -184,21 +147,23 @@ void Foam::functionObjects::parProfiling::report()
         const statData& reduceStats = times[0];
         const statData& allToAllStats = times[1];
 
-        scalar reduceAvg = reduceStats[2].second()/Pstream::nProcs();
+        double reduceAvg = reduceStats[2].first()/Pstream::nProcs();
-        scalar allToAllAvg = allToAllStats[2].second()/Pstream::nProcs();
+        double allToAllAvg = allToAllStats[2].first()/Pstream::nProcs();
 
         Info<< type() << ':' << nl
             << incrIndent
+
             << indent << "reduce    : avg = " << reduceAvg << 's' << nl
-            << indent << "            min = " << reduceStats[0].second()
+            << indent << "            min = " << reduceStats[0].first()
-            << "s (processor " << reduceStats[0].first() << ')' << nl
+            << "s (processor " << reduceStats[0].second() << ')' << nl
-            << indent << "            max = " << reduceStats[1].second()
+            << indent << "            max = " << reduceStats[1].first()
-            << "s (processor " << reduceStats[1].first() << ')' << nl
+            << "s (processor " << reduceStats[1].second() << ')' << nl
+
             << indent << "all-all   : avg = " << allToAllAvg << 's' << nl
-            << indent << "            min = " << allToAllStats[0].second()
+            << indent << "            min = " << allToAllStats[0].first()
-            << "s (processor " << allToAllStats[0].first() << ')' << nl
+            << "s (processor " << allToAllStats[0].second() << ')' << nl
-            << indent << "            max = " << allToAllStats[1].second()
+            << indent << "            max = " << allToAllStats[1].first()
-            << "s (processor " << allToAllStats[1].first() << ')'
+            << "s (processor " << allToAllStats[1].second() << ')'
             << decrIndent << endl;
     }
 }

src/functionObjects/utilities/syncObjects/syncObjects.C

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2020-2021 OpenCFD Ltd.
+    Copyright (C) 2020-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -96,8 +96,7 @@ void Foam::functionObjects::syncObjects::sync()
     );
 
 
-    const label nProcs = Pstream::nProcs(pBufs.comm());
+    for (const int proci : pBufs.allProcs())
-    for (label proci = 0; proci < nProcs; proci++)
     {
         // Get database to send
         const objectRegistry& sendObr = mappedPatchBase::subRegistry
@@ -123,7 +122,7 @@ void Foam::functionObjects::syncObjects::sync()
     // Start sending and receiving and block
     pBufs.finishedSends();
 
-    for (label proci = 0; proci < nProcs; proci++)
+    for (const int proci : pBufs.allProcs())
     {
         // Get database to receive data into
         const objectRegistry& receiveObr = mappedPatchBase::subRegistry

src/fvOptions/sources/derived/meanVelocityForce/patchMeanVelocityForce/patchMeanVelocityForce.C

@@ -6,6 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2015-2016 OpenFOAM Foundation
+    Copyright (C) 2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -71,15 +72,15 @@ Foam::scalar Foam::fv::patchMeanVelocityForce::magUbarAve
     const volVectorField& U
 ) const
 {
-    vector2D sumAmagUsumA
+    FixedList<scalar, 2> sumAmagUsumA(Zero);
-    (
+
+    sumAmagUsumA[0] +=
         sum
         (
             (flowDir_ & U.boundaryField()[patchi_])
           * mesh_.boundary()[patchi_].magSf()
-        ),
-        sum(mesh_.boundary()[patchi_].magSf())
         );
+    sumAmagUsumA[1] += sum(mesh_.boundary()[patchi_].magSf());
 
 
     // If the mean velocity force is applied to a cyclic patch
@@ -89,29 +90,29 @@ Foam::scalar Foam::fv::patchMeanVelocityForce::magUbarAve
 
     if (Pstream::parRun() && isA<cyclicPolyPatch>(patches[patchi_]))
     {
-        labelList processorCyclicPatches
+        for
         (
-            processorCyclicPolyPatch::patchIDs(patch_, patches)
+            const label patchi
-        );
+          : processorCyclicPolyPatch::patchIDs(patch_, patches)
-
+        )
-        forAll(processorCyclicPatches, pcpi)
         {
-            const label patchi = processorCyclicPatches[pcpi];
+            sumAmagUsumA[0] +=
-
-            sumAmagUsumA.x() +=
                 sum
                 (
                     (flowDir_ & U.boundaryField()[patchi])
                   * mesh_.boundary()[patchi].magSf()
                 );
-
+            sumAmagUsumA[1] += sum(mesh_.boundary()[patchi].magSf());
-            sumAmagUsumA.y() += sum(mesh_.boundary()[patchi].magSf());
         }
     }
 
-    mesh_.reduce(sumAmagUsumA, sumOp<vector2D>());
+    mesh_.reduce(sumAmagUsumA, sumOp<scalar>());
 
-    return sumAmagUsumA.x()/sumAmagUsumA.y();
+    return
+    (
+        sumAmagUsumA[0]
+      / stabilise(sumAmagUsumA[1], VSMALL)
+    );
 }
 
 

src/lagrangian/basic/Cloud/Cloud.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2017, 2020 OpenFOAM Foundation
-    Copyright (C) 2020-2021 OpenCFD Ltd.
+    Copyright (C) 2020-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -89,7 +89,7 @@ Foam::Cloud<ParticleType>::Cloud
     // Ask for the tetBasePtIs to trigger all processors to build
     // them, otherwise, if some processors have no particles then
     // there is a comms mismatch.
-    polyMesh_.tetBasePtIs();
+    (void)polyMesh_.tetBasePtIs();
 
     if (particles.size())
     {
@@ -163,54 +163,55 @@ void Foam::Cloud<ParticleType>::move
     // Which processors this processor is connected to
     const labelList& neighbourProcs = pData[Pstream::myProcNo()];
 
-    // Indexing from the processor number into the neighbourProcs list
-    labelList neighbourProcIndices(Pstream::nProcs(), -1);
-
-    forAll(neighbourProcs, i)
-    {
-        neighbourProcIndices[neighbourProcs[i]] = i;
-    }
-
     // Initialise the stepFraction moved for the particles
-    forAllIters(*this, pIter)
+    for (ParticleType& p : *this)
     {
-        pIter().reset();
+        p.reset();
     }
 
-    // List of lists of particles to be transferred for all of the
+    // Clear the global positions as these are about to change
-    // neighbour processors
-    List<IDLList<ParticleType>> particleTransferLists
-    (
-        neighbourProcs.size()
-    );
-
-    // List of destination processorPatches indices for all of the
-    // neighbour processors
-    List<DynamicList<label>> patchIndexTransferLists
-    (
-        neighbourProcs.size()
-    );
-
-    // Allocate transfer buffers
-    PstreamBuffers pBufs(Pstream::commsTypes::nonBlocking);
-
-    // Clear the global positions as there are about to change
     globalPositionsPtr_.clear();
 
+
+    // For v2112 and earlier: pre-assembled lists of particles
+    // to be transferred and target patch on a per processor basis.
+    // Apart from memory overhead of assembling the lists this adds
+    // allocations/de-allocation when building linked-lists.
+
+    // Now stream particle transfer tuples directly into PstreamBuffers.
+    // Use a local cache of UOPstream wrappers for the formatters
+    // (since there are potentially many particles being shifted about).
+
+
+    // Allocate transfer buffers,
+    // automatic clearStorage when UIPstream closes is disabled.
+    PstreamBuffers pBufs(Pstream::commsTypes::nonBlocking);
+    pBufs.allowClearRecv(false);
+
+    // Cache of opened UOPstream wrappers
+    PtrList<UOPstream> UOPstreamPtrs(Pstream::nProcs());
+
     // While there are particles to transfer
     while (true)
     {
-        particleTransferLists = IDLList<ParticleType>();
+        // Reset transfer buffers
-        forAll(patchIndexTransferLists, i)
+        pBufs.clear();
+
+        // Rewind existing streams
+        forAll(UOPstreamPtrs, proci)
         {
-            patchIndexTransferLists[i].clear();
+            auto* osptr = UOPstreamPtrs.get(proci);
+            if (osptr)
+            {
+                osptr->rewind();
+            }
         }
 
         // Loop over all particles
         for (ParticleType& p : *this)
         {
             // Move the particle
-            bool keepParticle = p.move(cloud, td, trackTime);
+            const bool keepParticle = p.move(cloud, td, trackTime);
 
             // If the particle is to be kept
             // (i.e. it hasn't passed through an inlet or outlet)
@@ -235,22 +236,27 @@ void Foam::Cloud<ParticleType>::move
 
                     const label patchi = p.patch();
 
-                    const label n = neighbourProcIndices
+                    const label toProci =
-                    [
-                        refCast<const processorPolyPatch>
                     (
-                            pbm[patchi]
+                        refCast<const processorPolyPatch>(pbm[patchi])
-                        ).neighbProcNo()
+                        .neighbProcNo()
-                    ];
+                    );
+
+                    // Get/create output stream
+                    auto* osptr = UOPstreamPtrs.get(toProci);
+                    if (!osptr)
+                    {
+                        osptr = new UOPstream(toProci, pBufs);
+                        UOPstreamPtrs.set(toProci, osptr);
+                    }
 
                     p.prepareForParallelTransfer();
 
-                    particleTransferLists[n].append(this->remove(&p));
+                    // Tuple: (patchi particle)
+                    (*osptr) << procPatchNeighbours[patchi] << p;
 
-                    patchIndexTransferLists[n].append
+                    // Can now remove from my list
-                    (
+                    deleteParticle(p);
-                        procPatchNeighbours[patchi]
-                    );
                 }
             }
             else
@@ -264,76 +270,32 @@ void Foam::Cloud<ParticleType>::move
             break;
         }
 
+        pBufs.finishedNeighbourSends(neighbourProcs);
 
-        // Clear transfer buffers
+        if (!returnReduce(pBufs.hasRecvData(), orOp<bool>()))
-        pBufs.clear();
-
-        // Stream into send buffers
-        forAll(particleTransferLists, i)
-        {
-            if (particleTransferLists[i].size())
-            {
-                UOPstream particleStream
-                (
-                    neighbourProcs[i],
-                    pBufs
-                );
-
-                particleStream
-                    << patchIndexTransferLists[i]
-                    << particleTransferLists[i];
-            }
-        }
-
-
-        // Start sending. Sets number of bytes transferred
-        labelList allNTrans(Pstream::nProcs());
-        pBufs.finishedSends(allNTrans);
-
-
-        bool transferred = false;
-
-        for (const label n : allNTrans)
-        {
-            if (n)
-            {
-                transferred = true;
-                break;
-            }
-        }
-        reduce(transferred, orOp<bool>());
-
-        if (!transferred)
         {
+            // No parcels to transfer
             break;
         }
 
         // Retrieve from receive buffers
-        for (const label neighbProci : neighbourProcs)
+        for (const label proci : neighbourProcs)
         {
-            label nRec = allNTrans[neighbProci];
+            if (pBufs.hasRecvData(proci))
-
-            if (nRec)
             {
-                UIPstream particleStream(neighbProci, pBufs);
+                UIPstream is(proci, pBufs);
 
-                labelList receivePatchIndex(particleStream);
+                // Read out each (patchi particle) tuple
-
+                while (!is.eof())
-                IDLList<ParticleType> newParticles
-                (
-                    particleStream,
-                    typename ParticleType::iNew(polyMesh_)
-                );
-
-                label pI = 0;
-
-                for (ParticleType& newp : newParticles)
                 {
-                    label patchi = procPatches[receivePatchIndex[pI++]];
+                    label patchi = pTraits<label>(is);
+                    auto* newp = new ParticleType(polyMesh_, is);
 
-                    newp.correctAfterParallelTransfer(patchi, td);
+                    // The real patch index
+                    patchi = procPatches[patchi];
 
-                    addParticle(newParticles.remove(&newp));
+                    (*newp).correctAfterParallelTransfer(patchi, td);
+                    addParticle(newp);
                 }
             }
         }
@@ -365,9 +327,9 @@ void Foam::Cloud<ParticleType>::autoMap(const mapPolyMesh& mapper)
     const vectorField& positions = globalPositionsPtr_();
 
     label i = 0;
-    forAllIters(*this, iter)
+    for (ParticleType& p : *this)
     {
-        iter().autoMap(positions[i], mapper);
+        p.autoMap(positions[i], mapper);
         ++i;
     }
 }
@@ -376,20 +338,19 @@ void Foam::Cloud<ParticleType>::autoMap(const mapPolyMesh& mapper)
 template<class ParticleType>
 void Foam::Cloud<ParticleType>::writePositions() const
 {
-    OFstream pObj
+    OFstream os
     (
         this->db().time().path()/this->name() + "_positions.obj"
     );
 
-    forAllConstIters(*this, pIter)
+    for (const ParticleType& p : *this)
     {
-        const ParticleType& p = pIter();
         const point position(p.position());
-        pObj<< "v " << position.x() << " " << position.y() << " "
+        os  << "v "
+            << position.x() << ' '
+            << position.y() << ' '
             << position.z() << nl;
     }
-
-    pObj.flush();
 }
 
 
@@ -402,13 +363,12 @@ void Foam::Cloud<ParticleType>::storeGlobalPositions() const
     // within autoMap, and this pre-processing would not be necessary.
 
     globalPositionsPtr_.reset(new vectorField(this->size()));
-
     vectorField& positions = globalPositionsPtr_();
 
     label i = 0;
-    forAllConstIters(*this, iter)
+    for (const ParticleType& p : *this)
     {
-        positions[i] = iter().position();
+        positions[i] = p.position();
         ++i;
     }
 }

src/lagrangian/intermediate/submodels/Kinematic/PatchInteractionModel/RecycleInteraction/RecycleInteraction.C

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2020 OpenCFD Ltd.
+    Copyright (C) 2020-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -205,10 +205,13 @@ void Foam::RecycleInteraction<CloudType>::postEvolve()
 {
     if (Pstream::parRun())
     {
-        // Relocate the recycled parcels into slots for each receiving processor
+        // See comments in Cloud::move() about transfer particles handling
-        List<IDLList<parcelType>> transferParcels(Pstream::nProcs());
+
-        List<DynamicList<scalar>> fractions(Pstream::nProcs());
+        // Allocate transfer buffers
-        List<DynamicList<label>> patchAddr(Pstream::nProcs());
+        PstreamBuffers pBufs(Pstream::commsTypes::nonBlocking);
+
+        // Cache of opened UOPstream wrappers
+        PtrList<UOPstream> UOPstreamPtrs(Pstream::nProcs());
 
         auto& rnd = this->owner().rndGen();
 
@@ -217,7 +220,7 @@ void Foam::RecycleInteraction<CloudType>::postEvolve()
             auto& patchParcels = recycledParcels_[addri];
             auto& injectionPatch = injectionPatchPtr_[addri];
 
-            forAllIters(patchParcels, pIter)
+            for (parcelType& p : patchParcels)
             {
                 // Choose a random location to insert the parcel
                 const scalar fraction01 = rnd.template sample01<scalar>();
@@ -225,101 +228,76 @@ void Foam::RecycleInteraction<CloudType>::postEvolve()
                 // Identify the processor that owns the location
                 const label toProci = injectionPatch.whichProc(fraction01);
 
-                // Store info in slot for target processor
+                // Get/create output stream
-                transferParcels[toProci].append(patchParcels.remove(pIter));
+                auto* osptr = UOPstreamPtrs.get(toProci);
-                fractions[toProci].append(fraction01);
+                if (!osptr)
-                patchAddr[toProci].append(addri);
-            }
-        }
-
-        // Set-up the sends
-        PstreamBuffers pBufs(Pstream::commsTypes::nonBlocking);
-
-        // Clear transfer buffers
-        pBufs.clear();
-
-        // Stream into send buffers
-        forAll(transferParcels, proci)
                 {
-            if (transferParcels[proci].size())
+                    osptr = new UOPstream(toProci, pBufs);
-            {
+                    UOPstreamPtrs.set(toProci, osptr);
-                UOPstream particleStream(proci,  pBufs);
+                }
 
-                particleStream
+                // Tuple: (address fraction particle)
-                    << transferParcels[proci]
+                (*osptr) << addri << fraction01 << p;
-                    << fractions[proci]
-                    << patchAddr[proci];
 
-                transferParcels[proci].clear();
+                // Can now remove from list and delete
+                delete(patchParcels.remove(&p));
             }
         }
 
-        // Start sending. Sets number of bytes transferred
+        pBufs.finishedSends();
-        labelList allNTrans(Pstream::nProcs());
+
-        pBufs.finishedSends(allNTrans);
+        if (!returnReduce(pBufs.hasRecvData(), orOp<bool>()))
-        bool transferred = false;
-        for (const label n : allNTrans)
         {
-            if (n)
+            // No parcels to recycle
-            {
-                transferred = true;
-                break;
-            }
-        }
-        reduce(transferred, orOp<bool>());
-        if (!transferred)
-        {
-            // No parcels to transfer
             return;
         }
 
         // Retrieve from receive buffers
-        for (label proci = 0; proci < Pstream::nProcs(); ++proci)
+        for (const int proci : pBufs.allProcs())
         {
-            if (allNTrans[proci])
+            if (pBufs.hasRecvData(proci))
             {
-                UIPstream particleStream(proci, pBufs);
+                UIPstream is(proci, pBufs);
-                IDLList<parcelType> newParticles
-                (
-                    particleStream,
-                    typename parcelType::iNew(this->owner().mesh())
-                );
-                scalarList fractions(particleStream);
-                labelList patchAddr(particleStream);
 
-                label parceli = 0;
+                // Read out each (address fraction particle) tuple
-                for (parcelType& newp : newParticles)
+                while (!is.eof())
                 {
+                    const label addri = pTraits<label>(is);
+                    const scalar fraction01 = pTraits<scalar>(is);
+                    auto* newp = new parcelType(this->owner().mesh(), is);
+
                     // Parcel to be recycled
                     vector newPosition;
                     label cellOwner;
                     label dummy;
-                    const label addri = patchAddr[parceli];
                     injectionPatchPtr_[addri].setPositionAndCell
                     (
                         mesh_,
-                        fractions[parceli],
+                        fraction01,
                         this->owner().rndGen(),
                         newPosition,
                         cellOwner,
                         dummy,
                         dummy
                     );
-                    newp.relocate(newPosition, cellOwner);
+                    newp->relocate(newPosition, cellOwner);
-                    newp.U() = this->owner().U()[cellOwner];
+                    newp->nParticle() *= recycleFraction_;
-                    newp.nParticle() *= recycleFraction_;
 
+                    // Assume parcel velocity is same as the carrier velocity
+                    newp->U() = this->owner().U()[cellOwner];
+
+                    // Injector ID
                     const label idx =
                     (
                         injIdToIndex_.size()
-                          ? injIdToIndex_.lookup(newp.typeId(), 0)
+                      ? injIdToIndex_.lookup(newp->typeId(), 0)
                       : 0
                     );
-                    ++nInjected_[addri][idx];
-                    massInjected_[addri][idx] += newp.nParticle()*newp.mass();
 
-                    this->owner().addParticle(newParticles.remove(&newp));
+                    ++nInjected_[addri][idx];
-                    ++parceli;
+                    massInjected_[addri][idx] += newp->nParticle()*newp->mass();
+
+                    this->owner().addParticle(newp);
                 }
             }
         }
@@ -328,8 +306,10 @@ void Foam::RecycleInteraction<CloudType>::postEvolve()
     {
         forAll(recycledParcels_, addri)
         {
-            forAllIters(recycledParcels_[addri], iter)
+            for (parcelType& p : recycledParcels_[addri])
             {
+                parcelType* newp = recycledParcels_[addri].remove(&p);
+
                 // Parcel to be recycled
                 vector newPosition;
                 label cellOwner;
@@ -345,13 +325,13 @@ void Foam::RecycleInteraction<CloudType>::postEvolve()
                 );
 
                 // Update parcel properties
-                parcelType* newp = recycledParcels_[addri].remove(iter);
                 newp->relocate(newPosition, cellOwner);
                 newp->nParticle() *= recycleFraction_;
 
                 // Assume parcel velocity is same as the carrier velocity
                 newp->U() = this->owner().U()[cellOwner];
 
+                // Injector ID
                 const label idx =
                 (
                     injIdToIndex_.size()
@@ -380,7 +360,7 @@ void Foam::RecycleInteraction<CloudType>::info(Ostream& os)
 
     forAll(nRemoved_, patchi)
     {
-        label lsd = nRemoved_[patchi].size();
+        const label lsd = nRemoved_[patchi].size();
         npr0[patchi].setSize(lsd, Zero);
         mpr0[patchi].setSize(lsd, Zero);
         npi0[patchi].setSize(lsd, Zero);

src/lagrangian/spray/clouds/Templates/SprayCloud/SprayCloud.H

@@ -35,11 +35,11 @@ Description
 
 \*---------------------------------------------------------------------------*/
 
-#ifndef SprayCloud_H
+#ifndef Foam_SprayCloud_H
-#define SprayCloud_H
+#define Foam_SprayCloud_H
 
 #include "sprayCloud.H"
-#include "SortableList.H"
+#include "SortList.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 

src/lagrangian/spray/clouds/Templates/SprayCloud/SprayCloudI.H

@@ -88,95 +88,73 @@ inline Foam::scalar Foam::SprayCloud<CloudType>::penetration
             << exit(FatalError);
     }
 
-    scalar distance = 0.0;
-
     const label nParcel = this->size();
-    globalIndex globalParcels(nParcel);
+    const globalIndex globalParcels(nParcel);
-    const label nParcelSum = globalParcels.totalSize();
+    const label nTotParcel = globalParcels.totalSize();
 
-    if (nParcelSum == 0)
+    if (nTotParcel == 0)
     {
-        return distance;
+        return 0;
     }
 
     // lists of parcels mass and distance from initial injection point
-    List<List<scalar>> procMass(Pstream::nProcs());
+    List<scalar> mass(nParcel);
-    List<List<scalar>> procDist(Pstream::nProcs());
+    List<scalar> dist(nParcel);
-
-    List<scalar>& mass = procMass[Pstream::myProcNo()];
-    List<scalar>& dist = procDist[Pstream::myProcNo()];
-
-    mass.setSize(nParcel);
-    dist.setSize(nParcel);
 
+    scalar mTotal = 0;
+    {
         label i = 0;
-    scalar mSum = 0.0;
         for (const parcelType& p : *this)
         {
             scalar m = p.nParticle()*p.mass();
             scalar d = mag(p.position() - p.position0());
-        mSum += m;
+            mTotal += m;
 
             mass[i] = m;
             dist[i] = d;
-
             ++i;
         }
+    }
+    // Total mass across all processors
+    reduce(mTotal, sumOp<scalar>());
 
-    // calculate total mass across all processors
+    scalar distance = 0;
-    reduce(mSum, sumOp<scalar>());
+    globalParcels.gatherInplace(mass);
-    Pstream::gatherList(procMass);
+    globalParcels.gatherInplace(dist);
-    Pstream::gatherList(procDist);
 
     if (Pstream::master())
     {
-        // flatten the mass lists
+        if (nTotParcel == 1)
-        List<scalar> allMass(nParcelSum, Zero);
-        SortableList<scalar> allDist(nParcelSum, Zero);
-        for (const int proci : Pstream::allProcs())
         {
-            SubList<scalar>
+            distance = dist[0];
-            (
-                allMass,
-                globalParcels.localSize(proci),
-                globalParcels.localStart(proci)
-            ) = procMass[proci];
-
-            // flatten the distance list
-            SubList<scalar>
-            (
-                allDist,
-                globalParcels.localSize(proci),
-                globalParcels.localStart(proci)
-            ) = procDist[proci];
         }
-
+        else
-        // sort allDist distances into ascending order
-        // note: allMass masses are left unsorted
-        allDist.sort();
-
-        if (nParcelSum > 1)
         {
-            const scalar mLimit = fraction*mSum;
+            // Distances - sored into ascending order
-            const labelList& indices = allDist.indices();
+            // Masses - leave unsorted
 
-            if (mLimit > (mSum - allMass[indices.last()]))
+            const SortList<scalar> sortedDist(dist);
+
+            const scalar mLimit = fraction*mTotal;
+            const labelList& indices = sortedDist.indices();
+
+            if (mLimit > (mTotal - mass[indices.last()]))
             {
-                distance = allDist.last();
+                distance = sortedDist.last();
             }
             else
             {
                 // assuming that 'fraction' is generally closer to 1 than 0,
                 // loop through in reverse distance order
-                const scalar mThreshold = (1.0 - fraction)*mSum;
+                const scalar mThreshold = (1.0 - fraction)*mTotal;
-                scalar mCurrent = 0.0;
+                scalar mCurrent = 0;
                 label i0 = 0;
 
                 forAllReverse(indices, i)
                 {
                     label indI = indices[i];
 
-                    mCurrent += allMass[indI];
+                    mCurrent += mass[indI];
 
                     if (mCurrent > mThreshold)
                     {
@@ -187,21 +165,20 @@ inline Foam::scalar Foam::SprayCloud<CloudType>::penetration
 
                 if (i0 == indices.size() - 1)
                 {
-                    distance = allDist.last();
+                    distance = sortedDist.last();
                 }
                 else
                 {
                     // linearly interpolate to determine distance
-                    scalar alpha = (mCurrent - mThreshold)/allMass[indices[i0]];
+                    scalar alpha = (mCurrent - mThreshold)/mass[indices[i0]];
                     distance =
-                        allDist[i0] + alpha*(allDist[i0+1] - allDist[i0]);
+                    (
+                        sortedDist[i0]
+                      + alpha*(sortedDist[i0+1] - sortedDist[i0])
+                    );
                 }
             }
         }
-        else
-        {
-            distance = allDist.first();
-        }
     }
 
     Pstream::scatter(distance);

src/lumpedPointMotion/state/lumpedPointState.C

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2016-2020 OpenCFD Ltd.
+    Copyright (C) 2016-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -403,53 +403,14 @@ bool Foam::lumpedPointState::readData
 
     if (Pstream::parRun())
     {
-        // Scatter master data using communication scheme
+        // Broadcast master data to everyone
 
-        const List<Pstream::commsStruct>& comms =
+        Pstream::scatter(points_);
-        (
+        Pstream::scatter(angles_);
-            (Pstream::nProcs() < Pstream::nProcsSimpleSum)
+        Pstream::scatter(degrees_);
-          ? Pstream::linearCommunication()
-          : Pstream::treeCommunication()
-        );
-
-        // Get my communication order
-        const Pstream::commsStruct& myComm = comms[Pstream::myProcNo()];
-
-        // Receive from up
-        if (myComm.above() != -1)
-        {
-            IPstream fromAbove
-            (
-                UPstream::commsTypes::scheduled,
-                myComm.above(),
-                0,
-                Pstream::msgType(),
-                Pstream::worldComm
-            );
-
-            fromAbove >> points_ >> angles_ >> degrees_;
-        }
-
-        // Send to downstairs neighbours
-        forAllReverse(myComm.below(), belowI)
-        {
-            OPstream toBelow
-            (
-                UPstream::commsTypes::scheduled,
-                myComm.below()[belowI],
-                0,
-                Pstream::msgType(),
-                Pstream::worldComm
-            );
-
-            toBelow << points_ << angles_ << degrees_;
-        }
-
-        rotationPtr_.reset(nullptr);
-
-        // MPI barrier
         Pstream::scatter(ok);
     }
+    rotationPtr_.reset(nullptr);
 
     return ok;
 }

src/meshTools/AMIInterpolation/AMIInterpolation/AMIInterpolation.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2017 OpenFOAM Foundation
-    Copyright (C) 2015-2021 OpenCFD Ltd.
+    Copyright (C) 2015-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -79,47 +79,37 @@ Foam::label Foam::AMIInterpolation::calcDistribution
     const primitivePatch& tgtPatch
 ) const
 {
+    // Either not parallel or no faces on any processor
     label proci = 0;
 
     if (Pstream::parRun())
     {
-        labelList facesPresentOnProc(Pstream::nProcs(), Zero);
+        const bitSet hasFaces
-        if ((srcPatch.size() > 0) || (tgtPatch.size() > 0))
+        (
-        {
+            UPstream::listGatherValues<bool>
-            facesPresentOnProc[Pstream::myProcNo()] = 1;
+            (
-        }
+                srcPatch.size() > 0 || tgtPatch.size() > 0
-        else
+            )
-        {
+        );
-            facesPresentOnProc[Pstream::myProcNo()] = 0;
-        }
 
-        Pstream::gatherList(facesPresentOnProc);
+        const auto nHaveFaces = hasFaces.count();
-        Pstream::scatterList(facesPresentOnProc);
 
-        label nHaveFaces = sum(facesPresentOnProc);
+        if (nHaveFaces == 1)
-
-        if (nHaveFaces > 1)
         {
-            proci = -1;
+            proci = hasFaces.find_first();
-            if (debug)
+            DebugInFunction
-            {
-                InfoInFunction
-                    << "AMI split across multiple processors" << endl;
-            }
-        }
-        else if (nHaveFaces == 1)
-        {
-            proci = facesPresentOnProc.find(1);
-            if (debug)
-            {
-                InfoInFunction
                 << "AMI local to processor" << proci << endl;
         }
-        }
+        else if (nHaveFaces > 1)
+        {
+            proci = -1;
+            DebugInFunction
+                << "AMI split across multiple processors" << endl;
         }
 
+        Pstream::broadcast(proci);
+    }
 
-    // Either not parallel or no faces on any processor
     return proci;
 }