ENH: consolidate UPstream::commsStruct code

- make sizing of commsStruct List demand-driven as well
  for more robustness, fewer unneeded allocations.

- fix potential latent bug with allBelow/allNotBelow for proc 0
  (linear communication).

ENH: remove unused/unusable UPstream::communicator optional parameter

- had constructor option to avoid constructing the MPI backend,
  but this is not useful and inconsistent with what the reset or
  destructor expect.

STYLE: local use of UPstream::communicator

- automatically frees communicator when it leaves scope

applications/test/parallel-comm2/Test-parallel-comm2.C

@@ -64,6 +64,7 @@ int main(int argc, char *argv[])
     argList::noBanner();
     argList::noCheckProcessorDirectories();
     argList::addBoolOption("verbose", "Set debug level");
+    argList::addBoolOption("print-tree", "Report tree(s) as graph");
     argList::addBoolOption("comm-split", "Test simple comm split");
     argList::addBoolOption("host-comm", "Test DIY host-comm split");
 
@@ -81,6 +82,8 @@ int main(int argc, char *argv[])
 
     #include "setRootCase.H"
 
+    const bool optPrintTree = args.found("print-tree");
+
     Info<< nl
         << "parallel:" << UPstream::parRun()
         << "nProcs = " << UPstream::nProcs()
@@ -102,6 +105,13 @@ int main(int argc, char *argv[])
     labelList subRanks;
     UPstream::communicator newComm;
 
+
+    if (UPstream::parRun() && optPrintTree)
+    {
+        Info<< "comms: " << UPstream::whichCommunication() << endl;
+        UPstream::printCommTree(UPstream::commWorld());
+    }
+
     if (!args.found("comm-split") && !args.found("host-comm"))
     {
         #if 1
@@ -377,12 +387,7 @@ int main(int argc, char *argv[])
 
         // From world to hostMaster
         const label hostMasterComm =
-            UPstream::allocateCommunicator
+            UPstream::allocateCommunicator(UPstream::commGlobal(), subRanks);
-            (
-                UPstream::commGlobal(),
-                subRanks,
-                true
-            );
 
 
         const label myHostId =
@@ -400,12 +405,7 @@ int main(int argc, char *argv[])
 
         // The intra-host ranks
         const label hostComm =
-            UPstream::allocateCommunicator
+            UPstream::allocateCommunicator(UPstream::commGlobal(), subRanks);
-            (
-                UPstream::commGlobal(),
-                subRanks,
-                true
-            );
 
         Pout<< nl << "[manual split]" << nl
             << nl << "Host comm with "
@@ -419,8 +419,8 @@ int main(int argc, char *argv[])
             << " sub-rank:" << UPstream::is_subrank(hostMasterComm)
             << nl;
 
-        UPstream::freeCommunicator(hostMasterComm, true);
+        UPstream::freeCommunicator(hostMasterComm);
-        UPstream::freeCommunicator(hostComm, true);
+        UPstream::freeCommunicator(hostComm);
     }
 
     Info<< "\nEnd\n" << endl;

applications/test/treeComms/Test-treeComms.C

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2022 OpenCFD Ltd.
+    Copyright (C) 2022-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -59,7 +59,7 @@ void printRecvCount_gatherList
     labelList nMesg;
     labelList nRecv;
 
-    if (UPstream::parRun() && np > 1 && Pstream::master(comm))
+    if (UPstream::parRun() && np > 1 && UPstream::master(comm))
     {
         nMesg.resize(np, Zero);
         nRecv.resize(np, Zero);
@@ -99,7 +99,7 @@ void printSendCount_scatterList
     labelList nMesg;
     labelList nSend;
 
-    if (UPstream::parRun() && np > 1 && Pstream::master(comm))
+    if (UPstream::parRun() && np > 1 && UPstream::master(comm))
     {
         nMesg.resize(np, Zero);
         nSend.resize(np, Zero);
@@ -139,7 +139,7 @@ void printWidths
     labelList maxBelow;
     labelList maxNotBelow;
 
-    if (UPstream::parRun() && np > 1 && Pstream::master(comm))
+    if (UPstream::parRun() && np > 1 && UPstream::master(comm))
     {
         maxBelow.resize(np, Zero);
         maxNotBelow.resize(np, Zero);
@@ -194,7 +194,7 @@ int main(int argc, char *argv[])
 
     // Info<< "allComms: " << comms << nl;
 
-    if (Pstream::master())
+    if (UPstream::master())
     {
         OFstream os("treeComm.dot");
 
@@ -204,9 +204,9 @@ int main(int argc, char *argv[])
         printConnection(os, 0, myComm.below());
         // Pout<< flatOutput(myComm.allBelow()) << nl;
 
-        for (const int proci : Pstream::subProcs())
+        for (const int proci : UPstream::subProcs())
         {
-            IPstream fromProc(Pstream::commsTypes::scheduled, proci);
+            IPstream fromProc(UPstream::commsTypes::scheduled, proci);
             labelList below(fromProc);
 
             printConnection(os, proci, below);

src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C

@@ -230,6 +230,10 @@ Foam::label Foam::UPstream::allocateCommunicator
 
     procIds.resize(numSubRanks);
 
+    // Sizing and filling are demand-driven
+    linearCommunication_[index].clear();
+    treeCommunication_[index].clear();
+
     if (doPstream && parRun())
     {
         allocatePstreamCommunicator(parentIndex, index);
@@ -249,13 +253,6 @@ Foam::label Foam::UPstream::allocateCommunicator
         /// }
     }
 
-    // In case communicator allocation adjusted procIDs_
-    numSubRanks = procIDs_[index].size();
-
-    // Size but do not fill structure - this is done on-the-fly
-    linearCommunication_[index] = List<commsStruct>(numSubRanks);
-    treeCommunication_[index] = List<commsStruct>(numSubRanks);
-
     return index;
 }
 
@@ -349,112 +346,40 @@ Foam::label Foam::UPstream::procNo
 }
 
 
-template<>
+const Foam::List<Foam::UPstream::commsStruct>&
-Foam::UPstream::commsStruct&
+Foam::UPstream::linearCommunication(const label communicator)
-Foam::UList<Foam::UPstream::commsStruct>::operator[](const label procID)
 {
-    UPstream::commsStruct& t = v_[procID];
+    if (linearCommunication_[communicator].empty())
-
-    if (t.allBelow().size() + t.allNotBelow().size() + 1 != size())
     {
-        // Not yet allocated
+        linearCommunication_[communicator] =
-
+            List<commsStruct>(UPstream::nProcs(communicator));
-        label above(-1);
-        labelList below;
-        labelList allBelow;
-
-        if (size() < UPstream::nProcsSimpleSum)
-        {
-            // Linear schedule
-
-            if (procID == 0)
-            {
-                below.setSize(size()-1);
-                for (label procI = 1; procI < size(); procI++)
-                {
-                    below[procI-1] = procI;
-                }
-            }
-            else
-            {
-                above = 0;
-            }
-        }
-        else
-        {
-            // Use tree like schedule. For 8 procs:
-            // (level 0)
-            //      0 receives from 1
-            //      2 receives from 3
-            //      4 receives from 5
-            //      6 receives from 7
-            // (level 1)
-            //      0 receives from 2
-            //      4 receives from 6
-            // (level 2)
-            //      0 receives from 4
-            //
-            // The sends/receives for all levels are collected per processor
-            // (one send per processor; multiple receives possible) creating
-            // a table:
-            //
-            // So per processor:
-            // proc     receives from   sends to
-            // ----     -------------   --------
-            //  0       1,2,4           -
-            //  1       -               0
-            //  2       3               0
-            //  3       -               2
-            //  4       5               0
-            //  5       -               4
-            //  6       7               4
-            //  7       -               6
-
-            label mod = 0;
-
-            for (label step = 1; step < size(); step = mod)
-            {
-                mod = step * 2;
-
-                if (procID % mod)
-                {
-                    above = procID - (procID % mod);
-                    break;
-                }
-                else
-                {
-                    for
-                    (
-                        label j = procID + step;
-                        j < size() && j < procID + mod;
-                        j += step
-                    )
-                    {
-                        below.append(j);
-                    }
-                    for
-                    (
-                        label j = procID + step;
-                        j < size() && j < procID + mod;
-                        j++
-                    )
-                    {
-                        allBelow.append(j);
-                    }
-                }
-            }
-        }
-        t = UPstream::commsStruct(size(), procID, above, below, allBelow);
     }
-    return t;
+
+    return linearCommunication_[communicator];
 }
 
 
-template<>
+const Foam::List<Foam::UPstream::commsStruct>&
-const Foam::UPstream::commsStruct&
+Foam::UPstream::treeCommunication(const label communicator)
-Foam::UList<Foam::UPstream::commsStruct>::operator[](const label procID) const
 {
-    return const_cast<UList<UPstream::commsStruct>&>(*this).operator[](procID);
+    if (treeCommunication_[communicator].empty())
+    {
+        treeCommunication_[communicator] =
+            List<commsStruct>(UPstream::nProcs(communicator));
+    }
+
+    return treeCommunication_[communicator];
+}
+
+
+void Foam::UPstream::printCommTree(const label communicator)
+{
+    const auto& comms = UPstream::whichCommunication(communicator);
+
+    if (UPstream::master(communicator))
+    {
+        commsStruct::printGraph(Info(), comms);
+    }
 }
 
 

src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H

@@ -98,17 +98,18 @@ public:
         {
             // Private Data
 
-                //- The procID of the processor directly above
+                //- The procID of the processor \em directly above
                 label above_;
 
-                //- The procIDs of all processors directly below
+                //- The procIDs of processors \em directly below
                 labelList below_;
 
-                //- procIDs of all processors below (so not just directly below)
+                //- The procIDs of all processors below myProcNo,
+                //- not just directly below
                 labelList allBelow_;
 
-                //- procIDs of all processors not below.
+                //- The procIDs of all processors not below myProcNo
-                //  Inverse set of allBelow_ without myProcNo.
+                //  (inverse of allBelow_ without myProcNo)
                 labelList allNotBelow_;
 
 
@@ -119,19 +120,19 @@ public:
                 //- Default construct with above == -1
                 commsStruct() noexcept : above_(-1) {}
 
-                //- Construct from components
+                //- Move construct from components
                 commsStruct
                 (
                     const label above,
-                    const labelUList& below,
+                    labelList&& below,
-                    const labelUList& allBelow,
+                    labelList&& allBelow,
-                    const labelUList& allNotBelow
+                    labelList&& allNotBelow
                 );
 
-                //- Construct from components; construct allNotBelow_
+                //- Copy construct from below, allBelow components
                 commsStruct
                 (
-                    const label nProcs,
+                    const label numProcs,
                     const label myProcID,
                     const label above,
                     const labelUList& below,
@@ -141,29 +142,31 @@ public:
 
             // Member Functions
 
-                //- Reset to default constructed state
+                //- Print un-directed graph in graphviz dot format
-                void clear();
+                static void printGraph
+                (
+                    Ostream& os,
+                    const UList<UPstream::commsStruct>& comms,
+                    const label proci = 0  // starting node
+                );
 
-                //- The procID of the processor directly above
-                label above() const noexcept
-                {
-                    return above_;
-                }
 
-                //- The procIDs of all processors directly below
+            // Access
-                const labelList& below() const noexcept
+
-                {
+                //- The number of processors addressed by the structure
-                    return below_;
+                label nProcs() const;
-                }
+
+                //- The procID of the processor \em directly above
+                label above() const noexcept { return above_; }
+
+                //- The procIDs of the processors \em directly below
+                const labelList& below() const noexcept { return below_; }
 
                 //- The procIDs of all processors below
                 //- (so not just directly below)
-                const labelList& allBelow() const noexcept
+                const labelList& allBelow() const noexcept { return allBelow_; }
-                {
-                    return allBelow_;
-                }
 
-                //- The procIDs of all processors that are above.
+                //- The procIDs of all processors not below myProcNo.
                 //- The inverse set of allBelow without myProcNo.
                 const labelList& allNotBelow() const noexcept
                 {
@@ -171,14 +174,20 @@ public:
                 }
 
 
-            // Member Operators
+            // Edit
+
+                //- Reset to default constructed state
+                void reset();
+
+                //- Reset with automatic linear/tree selection
+                void reset(const label procID, const label numProcs);
+
+
+            // Member / Friend Operators
 
                 bool operator==(const commsStruct&) const;
                 bool operator!=(const commsStruct&) const;
 
-
-             // Ostream Operator
-
                 friend Ostream& operator<<(Ostream&, const commsStruct&);
         };
 
@@ -335,6 +344,9 @@ public:
             return (communicator > worldComm && communicator > selfComm);
         }
 
+        //- Debugging: print the communication tree
+        static void printCommTree(const label communicator);
+
 
     // Constructors
 
@@ -391,24 +403,21 @@ public:
             //- Move construct, takes ownership
             communicator(communicator&& c) : comm_(c.comm_) { c.comm_ = -1; }
 
-            //- Allocate a communicator based on given parent
+            //- Allocate communicator for sub-ranks on given parent
             communicator
             (
                 //! The parent communicator
-                const label parent,
+                const label parentComm,
 
-                //! The sub-ranks of parent to use (ignore negative values)
+                //! The sub-ranks of parent to use (negative values ignored)
-                const labelUList& subRanks,
+                const labelUList& subRanks
-
-                //! Call allocatePstreamCommunicator
-                const bool doPstream = true
             )
             :
-                comm_(allocateCommunicator(parent, subRanks, doPstream))
+                comm_(UPstream::allocateCommunicator(parentComm, subRanks))
             {}
 
             //- Free allocated communicator and group
-            ~communicator() { freeCommunicator(comm_); }
+            ~communicator() { UPstream::freeCommunicator(comm_); }
 
             //- True if communicator is non-negative (ie, was allocated)
             bool good() const noexcept { return (comm_ >= 0); }
@@ -417,19 +426,19 @@ public:
             label comm() const noexcept { return comm_; }
 
             //- Free allocated communicator and group
-            void reset() { freeCommunicator(comm_); comm_ = -1; }
+            void reset() { UPstream::freeCommunicator(comm_); comm_ = -1; }
 
             //- Allocate with subRanks of parent communicator
             void reset(label parent, const labelUList& subRanks)
             {
-                freeCommunicator(comm_);
+                UPstream::freeCommunicator(comm_);
-                comm_ = allocateCommunicator(parent, subRanks);
+                comm_ = UPstream::allocateCommunicator(parent, subRanks);
             }
 
             //- Take ownership, free allocated communicator and group.
             void reset(communicator&& c)
             {
-                if (comm_ != c.comm_) freeCommunicator(comm_);
+                if (comm_ != c.comm_) UPstream::freeCommunicator(comm_);
                 comm_ = c.comm_;
                 c.comm_ = -1;
             }
@@ -730,22 +739,17 @@ public:
         }
 
         //- Communication schedule for linear all-to-master (proc 0)
-        static const List<commsStruct>& linearCommunication
+        static const List<commsStruct>&
+        linearCommunication
         (
             const label communicator = worldComm
-        )
+        );
-        {
-            return linearCommunication_[communicator];
-        }
 
         //- Communication schedule for tree all-to-master (proc 0)
         static const List<commsStruct>& treeCommunication
         (
             const label communicator = worldComm
-        )
+        );
-        {
-            return treeCommunication_[communicator];
-        }
 
         //- Communication schedule for linear/tree all-to-master (proc 0).
         //- Chooses based on the value of UPstream::nProcsSimpleSum
@@ -757,8 +761,8 @@ public:
             return
             (
                 nProcs(communicator) < nProcsSimpleSum
-              ? linearCommunication_[communicator]
+              ? linearCommunication(communicator)
-              : treeCommunication_[communicator]
+              : treeCommunication(communicator)
             );
         }
 

src/OpenFOAM/db/IOstreams/Pstreams/UPstreamCommsStruct.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2016 OpenFOAM Foundation
-    Copyright (C) 2021-2022 OpenCFD Ltd.
+    Copyright (C) 2021-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -33,21 +33,21 @@ License
 Foam::UPstream::commsStruct::commsStruct
 (
     const label above,
-    const labelUList& below,
+    labelList&& below,
-    const labelUList& allBelow,
+    labelList&& allBelow,
-    const labelUList& allNotBelow
+    labelList&& allNotBelow
 )
 :
     above_(above),
-    below_(below),
+    below_(std::move(below)),
-    allBelow_(allBelow),
+    allBelow_(std::move(allBelow)),
-    allNotBelow_(allNotBelow)
+    allNotBelow_(std::move(allNotBelow))
 {}
 
 
 Foam::UPstream::commsStruct::commsStruct
 (
-    const label nProcs,
+    const label numProcs,
     const label myProcID,
     const label above,
     const labelUList& below,
@@ -57,33 +57,114 @@ Foam::UPstream::commsStruct::commsStruct
     above_(above),
     below_(below),
     allBelow_(allBelow),
-    allNotBelow_(nProcs - allBelow.size() - 1)
+    allNotBelow_(numProcs - allBelow.size() - 1)
 {
-    boolList inBelow(nProcs, false);
+    List<bool> isNotBelow(numProcs, true);
 
-    forAll(allBelow, belowI)
+    // Exclude self
+    isNotBelow[myProcID] = false;
+
+    // Exclude allBelow
+    for (const label proci : allBelow)
     {
-        inBelow[allBelow[belowI]] = true;
+        isNotBelow[proci] = false;
     }
 
-    label notI = 0;
+    // Compacting to obtain allNotBelow_
-    forAll(inBelow, proci)
+    label nNotBelow = 0;
+    forAll(isNotBelow, proci)
     {
-        if ((proci != myProcID) && !inBelow[proci])
+        if (isNotBelow[proci])
         {
-            allNotBelow_[notI++] = proci;
+            allNotBelow_[nNotBelow++] = proci;
         }
     }
-    if (notI != allNotBelow_.size())
+
+    if (nNotBelow != allNotBelow_.size())
     {
-        FatalErrorInFunction << "problem!" << Foam::abort(FatalError);
+        FatalErrorInFunction
+            << "Problem: " << nNotBelow << " != " << allNotBelow_.size() << nl
+            << Foam::abort(FatalError);
+    }
+}
+
+
+// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * //
+
+// This outputs as depth-first, but graphviz sorts that for us
+void Foam::UPstream::commsStruct::printGraph
+(
+    Ostream& os,
+    const UList<UPstream::commsStruct>& comms,
+    const label proci
+)
+{
+    // if (proci >= comms.size()) return;  // Extreme safety!
+
+    const auto& below = comms[proci].below();
+
+    if (proci == 0)
+    {
+        os << nl << "// communication graph:" << nl;
+        os.beginBlock("graph");
+
+        if (below.empty())
+        {
+            // A graph with a single-node (eg, self-comm)
+            os << indent << proci << nl;
+        }
+    }
+
+    int pos = 0;
+
+    for (const label nbrProci : below)
+    {
+        if (pos)
+        {
+            os << "  ";
+        }
+        else
+        {
+            os << indent;
+        }
+        os << proci << " -- " << nbrProci;
+
+        if (++pos >= 4)  // Max 4 items per line
+        {
+            pos = 0;
+            os << nl;
+        }
+    }
+
+    if (pos)
+    {
+        os << nl;
+    }
+
+    for (const label nbrProci : below)
+    {
+        // if (proci == nbrProci) continue;  // Extreme safety!
+        printGraph(os, comms, nbrProci);
+    }
+
+    if (proci == 0)
+    {
+        os.endBlock();
+
+        os << "// end graph" << nl;
     }
 }
 
 
 // * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
 
-void Foam::UPstream::commsStruct::clear()
+Foam::label Foam::UPstream::commsStruct::nProcs() const
+{
+    return (1 + allBelow_.size() + allNotBelow_.size());
+}
+
+
+void Foam::UPstream::commsStruct::reset()
 {
     above_ = -1;
     below_.clear();
@@ -92,6 +173,127 @@ void Foam::UPstream::commsStruct::clear()
 }
 
 
+void Foam::UPstream::commsStruct::reset
+(
+    const label procID,
+    const label numProcs
+)
+{
+    reset();
+
+    label above(-1);
+    DynamicList<label> below;
+    DynamicList<label> allBelow;
+
+    if (numProcs < UPstream::nProcsSimpleSum)
+    {
+        // Linear schedule
+
+        if (procID == 0)
+        {
+            below = identity(numProcs-1, 1);
+            allBelow = below;
+        }
+        else
+        {
+            above = 0;
+        }
+    }
+    else
+    {
+        // Use tree like schedule. For 8 procs:
+        // (level 0)
+        //      0 receives from 1
+        //      2 receives from 3
+        //      4 receives from 5
+        //      6 receives from 7
+        // (level 1)
+        //      0 receives from 2
+        //      4 receives from 6
+        // (level 2)
+        //      0 receives from 4
+        //
+        // The sends/receives for all levels are collected per processor
+        // (one send per processor; multiple receives possible) creating
+        // a table:
+        //
+        // So per processor:
+        // proc     receives from   sends to
+        // ----     -------------   --------
+        //  0       1,2,4           -
+        //  1       -               0
+        //  2       3               0
+        //  3       -               2
+        //  4       5               0
+        //  5       -               4
+        //  6       7               4
+        //  7       -               6
+
+        label mod = 0;
+
+        for (label step = 1; step < numProcs; step = mod)
+        {
+            mod = step * 2;
+
+            if (procID % mod)
+            {
+                above = procID - (procID % mod);
+                break;
+            }
+            else
+            {
+                for
+                (
+                    label j = procID + step;
+                    j < numProcs && j < procID + mod;
+                    j += step
+                )
+                {
+                    below.push_back(j);
+                }
+                for
+                (
+                    label j = procID + step;
+                    j < numProcs && j < procID + mod;
+                    j++
+                )
+                {
+                    allBelow.push_back(j);
+                }
+            }
+        }
+    }
+
+    *this = UPstream::commsStruct(numProcs, procID, above, below, allBelow);
+}
+
+
+// * * * * * * * * * * * * * * * Specializations * * * * * * * * * * * * * * //
+
+template<>
+Foam::UPstream::commsStruct&
+Foam::UList<Foam::UPstream::commsStruct>::operator[](const label procID)
+{
+    auto& val = this->v_[procID];   // or this->data()[procID]
+
+    if (val.nProcs() != size())
+    {
+        // Create/update
+        val.reset(procID, size());
+    }
+
+    return val;
+}
+
+
+template<>
+const Foam::UPstream::commsStruct&
+Foam::UList<Foam::UPstream::commsStruct>::operator[](const label procID) const
+{
+    return const_cast<UList<UPstream::commsStruct>&>(*this).operator[](procID);
+}
+
+
 // * * * * * * * * * * * * * * * Member Operators  * * * * * * * * * * * * * //
 
 bool Foam::UPstream::commsStruct::operator==(const commsStruct& comm) const
@@ -100,8 +302,8 @@ bool Foam::UPstream::commsStruct::operator==(const commsStruct& comm) const
     (
         (above_ == comm.above())
      && (below_ == comm.below())
-     && (allBelow_ == allBelow())
+     // && (allBelow_ == comm.allBelow())
-     && (allNotBelow_ == allNotBelow())
+     // && (allNotBelow_ == comm.allNotBelow())
     );
 }
 
@@ -116,10 +318,10 @@ bool Foam::UPstream::commsStruct::operator!=(const commsStruct& comm) const
 
 Foam::Ostream& Foam::operator<<(Ostream& os, const UPstream::commsStruct& comm)
 {
-    os  << comm.above_ << token::SPACE
+    os  << comm.above() << nl << token::SPACE << token::SPACE;
-        << comm.below_ << token::SPACE
+    comm.below().writeList(os) << nl << token::SPACE << token::SPACE;
-        << comm.allBelow_ << token::SPACE
+    comm.allBelow().writeList(os) << nl << token::SPACE << token::SPACE;
-        << comm.allNotBelow_;
+    comm.allNotBelow().writeList(os);
 
     os.check(FUNCTION_NAME);
     return os;

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/eagerGAMGProcAgglomeration/eagerGAMGProcAgglomeration.C

@@ -65,10 +65,7 @@ Foam::eagerGAMGProcAgglomeration::
 {
     forAllReverse(comms_, i)
     {
-        if (comms_[i] != -1)
+        UPstream::freeCommunicator(comms_[i]);
-        {
-            UPstream::freeCommunicator(comms_[i]);
-        }
     }
 }
 

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/manualGAMGProcAgglomeration/manualGAMGProcAgglomeration.C

@@ -64,10 +64,7 @@ Foam::manualGAMGProcAgglomeration::
 {
     forAllReverse(comms_, i)
     {
-        if (comms_[i] != -1)
+        UPstream::freeCommunicator(comms_[i]);
-        {
-            UPstream::freeCommunicator(comms_[i]);
-        }
     }
 }
 

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/masterCoarsestGAMGProcAgglomeration/masterCoarsestGAMGProcAgglomeration.C

@@ -104,10 +104,7 @@ Foam::masterCoarsestGAMGProcAgglomeration::
 {
     forAllReverse(comms_, i)
     {
-        if (comms_[i] != -1)
+        UPstream::freeCommunicator(comms_[i]);
-        {
-            UPstream::freeCommunicator(comms_[i]);
-        }
     }
 }
 

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/procFacesGAMGProcAgglomeration/procFacesGAMGProcAgglomeration.C

@@ -149,10 +149,10 @@ Foam::procFacesGAMGProcAgglomeration::processorAgglomeration
     const lduMesh& mesh
 ) const
 {
-    label singleCellMeshComm = UPstream::allocateCommunicator
+    UPstream::communicator singleCellMeshComm
     (
         mesh.comm(),
-        labelList(1, Zero)   // only processor 0
+        labelList(Foam::one{}, 0)   // only processor 0
     );
 
     scalarField faceWeights;
@@ -160,14 +160,14 @@ Foam::procFacesGAMGProcAgglomeration::processorAgglomeration
     (
         singleCellMesh
         (
-            singleCellMeshComm,
+            singleCellMeshComm.comm(),
             mesh,
             faceWeights
         )
     );
 
-    tmp<labelField> tfineToCoarse(new labelField(0));
+    auto tfineToCoarse = tmp<labelField>::New();
-    labelField& fineToCoarse = tfineToCoarse.ref();
+    auto& fineToCoarse = tfineToCoarse.ref();
 
     if (singleCellMeshPtr)
     {
@@ -197,7 +197,7 @@ Foam::procFacesGAMGProcAgglomeration::processorAgglomeration
     }
 
     Pstream::broadcast(fineToCoarse, mesh.comm());
-    UPstream::freeCommunicator(singleCellMeshComm);
+    singleCellMeshComm.reset();
 
     return tfineToCoarse;
 }
@@ -234,10 +234,7 @@ Foam::procFacesGAMGProcAgglomeration::~procFacesGAMGProcAgglomeration()
 {
     forAllReverse(comms_, i)
     {
-        if (comms_[i] != -1)
+        UPstream::freeCommunicator(comms_[i]);
-        {
-            UPstream::freeCommunicator(comms_[i]);
-        }
     }
 }
 

src/OpenFOAM/meshes/polyMesh/globalMeshData/globalMeshData.C

@@ -2714,13 +2714,14 @@ void Foam::globalMeshData::updateMesh()
 
     // *** Temporary hack to avoid problems with overlapping communication
     // *** between these reductions and the calculation of deltaCoeffs
-    //const label comm = UPstream::worldComm + 1;
+
-    const label comm = UPstream::allocateCommunicator
+    UPstream::communicator dupComm
     (
         UPstream::worldComm,
-        identity(UPstream::nProcs(UPstream::worldComm)),
+        identity(UPstream::nProcs(UPstream::worldComm))
-        true
     );
+
+    const label comm = dupComm.comm();
     const label oldWarnComm = UPstream::commWarn(comm);
 
 
@@ -2760,8 +2761,8 @@ void Foam::globalMeshData::updateMesh()
     );
 
     // Restore communicator settings
-    UPstream::freeCommunicator(comm);
     UPstream::commWarn(oldWarnComm);
+    dupComm.reset();
 
     if (debug)
     {

src/Pstream/mpi/UPstream.C

@@ -335,12 +335,7 @@ bool Foam::UPstream::init(int& argc, char**& argv, const bool needsThread)
 
         // Allocate new communicator with globalComm as its parent
         const label subComm =
-            UPstream::allocateCommunicator
+            UPstream::allocateCommunicator(UPstream::globalComm, subRanks);
-            (
-                UPstream::globalComm,  // parent
-                subRanks,
-                true
-            );
 
 
         // Override worldComm

src/meshTools/multiWorld/multiWorldConnectionsObject.C

@@ -157,13 +157,12 @@ Foam::label Foam::multiWorldConnections::createCommunicator(const edge& worlds)
     {
         if (worlds.found(worldIDs[proci]))
         {
-            subRanks.append(proci);
+            subRanks.push_back(proci);
         }
     }
 
     // Allocate new communicator with global world
-    comm =
+    comm = UPstream::allocateCommunicator(UPstream::commGlobal(), subRanks);
-        UPstream::allocateCommunicator(UPstream::commGlobal(), subRanks, true);
 
     if (debug & 2)
     {