zhyang-dev/openfoam
1
0
Fork 0
mirror of https://develop.openfoam.com/Development/openfoam.git synced 2025-11-28 03:28:01 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

ENH: consolidate UPstream::commsStruct code

Browse Source 
- make sizing of commsStruct List demand-driven as well
  for more robustness, fewer unneeded allocations.

- fix potential latent bug with allBelow/allNotBelow for proc 0
  (linear communication).

ENH: remove unused/unusable UPstream::communicator optional parameter

- had constructor option to avoid constructing the MPI backend,
  but this is not useful and inconsistent with what the reset or
  destructor expect.

STYLE: local use of UPstream::communicator

- automatically frees communicator when it leaves scope
This commit is contained in:
Mark Olesen
2023-04-14 15:00:03 +02:00
parent b277b913cf
commit 9d3427e0b4
12 changed files with 358 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
applications/test/parallel-comm2/Test-parallel-comm2.C
View File

@ -64,6 +64,7 @@ int main(int argc, char *argv[])
argList::noBanner();
argList::noCheckProcessorDirectories();
argList::addBoolOption("verbose", "Set debug level");
argList::addBoolOption("print-tree", "Report tree(s) as graph");
argList::addBoolOption("comm-split", "Test simple comm split");
argList::addBoolOption("host-comm", "Test DIY host-comm split");
@ -81,6 +82,8 @@ int main(int argc, char *argv[])
#include "setRootCase.H"
const bool optPrintTree = args.found("print-tree");
Info<< nl
<< "parallel:" << UPstream::parRun()
<< "nProcs = " << UPstream::nProcs()
@ -102,6 +105,13 @@ int main(int argc, char *argv[])
labelList subRanks;
UPstream::communicator newComm;
if (UPstream::parRun() && optPrintTree)
{
Info<< "comms: " << UPstream::whichCommunication() << endl;
UPstream::printCommTree(UPstream::commWorld());
}
if (!args.found("comm-split") && !args.found("host-comm"))
{
#if 1
@ -377,12 +387,7 @@ int main(int argc, char *argv[])
// From world to hostMaster
const label hostMasterComm =
UPstream::allocateCommunicator
(
UPstream::commGlobal(),
subRanks,
true
);
UPstream::allocateCommunicator(UPstream::commGlobal(), subRanks);
const label myHostId =
@ -400,12 +405,7 @@ int main(int argc, char *argv[])
// The intra-host ranks
const label hostComm =
UPstream::allocateCommunicator
(
UPstream::commGlobal(),
subRanks,
true
);
UPstream::allocateCommunicator(UPstream::commGlobal(), subRanks);
Pout<< nl << "[manual split]" << nl
<< nl << "Host comm with "
@ -419,8 +419,8 @@ int main(int argc, char *argv[])
<< " sub-rank:" << UPstream::is_subrank(hostMasterComm)
<< nl;
UPstream::freeCommunicator(hostMasterComm, true);
UPstream::freeCommunicator(hostComm, true);
UPstream::freeCommunicator(hostMasterComm);
UPstream::freeCommunicator(hostComm);
}
Info<< "\nEnd\n" << endl;

14
applications/test/treeComms/Test-treeComms.C
View File

@ -5,7 +5,7 @@
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2022 OpenCFD Ltd.
Copyright (C) 2022-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -59,7 +59,7 @@ void printRecvCount_gatherList
labelList nMesg;
labelList nRecv;
if (UPstream::parRun() && np > 1 && Pstream::master(comm))
if (UPstream::parRun() && np > 1 && UPstream::master(comm))
{
nMesg.resize(np, Zero);
nRecv.resize(np, Zero);
@ -99,7 +99,7 @@ void printSendCount_scatterList
labelList nMesg;
labelList nSend;
if (UPstream::parRun() && np > 1 && Pstream::master(comm))
if (UPstream::parRun() && np > 1 && UPstream::master(comm))
{
nMesg.resize(np, Zero);
nSend.resize(np, Zero);
@ -139,7 +139,7 @@ void printWidths
labelList maxBelow;
labelList maxNotBelow;
if (UPstream::parRun() && np > 1 && Pstream::master(comm))
if (UPstream::parRun() && np > 1 && UPstream::master(comm))
{
maxBelow.resize(np, Zero);
maxNotBelow.resize(np, Zero);
@ -194,7 +194,7 @@ int main(int argc, char *argv[])
// Info<< "allComms: " << comms << nl;
if (Pstream::master())
if (UPstream::master())
{
OFstream os("treeComm.dot");
@ -204,9 +204,9 @@ int main(int argc, char *argv[])
printConnection(os, 0, myComm.below());
// Pout<< flatOutput(myComm.allBelow()) << nl;
for (const int proci : Pstream::subProcs())
for (const int proci : UPstream::subProcs())
{
IPstream fromProc(Pstream::commsTypes::scheduled, proci);
IPstream fromProc(UPstream::commsTypes::scheduled, proci);
labelList below(fromProc);
printConnection(os, proci, below);

139
src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C
View File

@ -230,6 +230,10 @@ Foam::label Foam::UPstream::allocateCommunicator
procIds.resize(numSubRanks);
// Sizing and filling are demand-driven
linearCommunication_[index].clear();
treeCommunication_[index].clear();
if (doPstream && parRun())
{
allocatePstreamCommunicator(parentIndex, index);
@ -249,13 +253,6 @@ Foam::label Foam::UPstream::allocateCommunicator
/// }
}
// In case communicator allocation adjusted procIDs_
numSubRanks = procIDs_[index].size();
// Size but do not fill structure - this is done on-the-fly
linearCommunication_[index] = List<commsStruct>(numSubRanks);
treeCommunication_[index] = List<commsStruct>(numSubRanks);
return index;
}
@ -349,112 +346,40 @@ Foam::label Foam::UPstream::procNo
}
template<>
Foam::UPstream::commsStruct&
Foam::UList<Foam::UPstream::commsStruct>::operator[](const label procID)
const Foam::List<Foam::UPstream::commsStruct>&
Foam::UPstream::linearCommunication(const label communicator)
{
UPstream::commsStruct& t = v_[procID];
if (linearCommunication_[communicator].empty())
{
linearCommunication_[communicator] =
List<commsStruct>(UPstream::nProcs(communicator));
}
if (t.allBelow().size() + t.allNotBelow().size() + 1 != size())
{
// Not yet allocated
label above(-1);
labelList below;
labelList allBelow;
if (size() < UPstream::nProcsSimpleSum)
{
// Linear schedule
if (procID == 0)
{
below.setSize(size()-1);
for (label procI = 1; procI < size(); procI++)
{
below[procI-1] = procI;
}
}
else
{
above = 0;
}
}
else
{
// Use tree like schedule. For 8 procs:
// (level 0)
// 0 receives from 1
// 2 receives from 3
// 4 receives from 5
// 6 receives from 7
// (level 1)
// 0 receives from 2
// 4 receives from 6
// (level 2)
// 0 receives from 4
//
// The sends/receives for all levels are collected per processor
// (one send per processor; multiple receives possible) creating
// a table:
//
// So per processor:
// proc receives from sends to
// ---- ------------- --------
// 0 1,2,4 -
// 1 - 0
// 2 3 0
// 3 - 2
// 4 5 0
// 5 - 4
// 6 7 4
// 7 - 6
label mod = 0;
for (label step = 1; step < size(); step = mod)
{
mod = step * 2;
if (procID % mod)
{
above = procID - (procID % mod);
break;
}
else
{
for
(
label j = procID + step;
j < size() && j < procID + mod;
j += step
)
{
below.append(j);
}
for
(
label j = procID + step;
j < size() && j < procID + mod;
j++
)
{
allBelow.append(j);
}
}
}
}
t = UPstream::commsStruct(size(), procID, above, below, allBelow);
}
return t;
return linearCommunication_[communicator];
}
template<>
const Foam::UPstream::commsStruct&
Foam::UList<Foam::UPstream::commsStruct>::operator[](const label procID) const
const Foam::List<Foam::UPstream::commsStruct>&
Foam::UPstream::treeCommunication(const label communicator)
{
return const_cast<UList<UPstream::commsStruct>&>(*this).operator[](procID);
if (treeCommunication_[communicator].empty())
{
treeCommunication_[communicator] =
List<commsStruct>(UPstream::nProcs(communicator));
}
return treeCommunication_[communicator];
}
void Foam::UPstream::printCommTree(const label communicator)
{
const auto& comms = UPstream::whichCommunication(communicator);
if (UPstream::master(communicator))
{
commsStruct::printGraph(Info(), comms);
}
}

116
src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H
View File

@ -98,17 +98,18 @@ public:
{
// Private Data
//- The procID of the processor directly above
//- The procID of the processor \em directly above
label above_;
//- The procIDs of all processors directly below
//- The procIDs of processors \em directly below
labelList below_;
//- procIDs of all processors below (so not just directly below)
//- The procIDs of all processors below myProcNo,
//- not just directly below
labelList allBelow_;
//- procIDs of all processors not below.
// Inverse set of allBelow_ without myProcNo.
//- The procIDs of all processors not below myProcNo
// (inverse of allBelow_ without myProcNo)
labelList allNotBelow_;
@ -119,19 +120,19 @@ public:
//- Default construct with above == -1
commsStruct() noexcept : above_(-1) {}
//- Construct from components
//- Move construct from components
commsStruct
(
const label above,
const labelUList& below,
const labelUList& allBelow,
const labelUList& allNotBelow
labelList&& below,
labelList&& allBelow,
labelList&& allNotBelow
);
//- Construct from components; construct allNotBelow_
//- Copy construct from below, allBelow components
commsStruct
(
const label nProcs,
const label numProcs,
const label myProcID,
const label above,
const labelUList& below,
@ -141,29 +142,31 @@ public:
// Member Functions
//- Reset to default constructed state
void clear();
//- Print un-directed graph in graphviz dot format
static void printGraph
(
Ostream& os,
const UList<UPstream::commsStruct>& comms,
const label proci = 0 // starting node
);
//- The procID of the processor directly above
label above() const noexcept
{
return above_;
}
//- The procIDs of all processors directly below
const labelList& below() const noexcept
{
return below_;
}
// Access
//- The number of processors addressed by the structure
label nProcs() const;
//- The procID of the processor \em directly above
label above() const noexcept { return above_; }
//- The procIDs of the processors \em directly below
const labelList& below() const noexcept { return below_; }
//- The procIDs of all processors below
//- (so not just directly below)
const labelList& allBelow() const noexcept
{
return allBelow_;
}
const labelList& allBelow() const noexcept { return allBelow_; }
//- The procIDs of all processors that are above.
//- The procIDs of all processors not below myProcNo.
//- The inverse set of allBelow without myProcNo.
const labelList& allNotBelow() const noexcept
{
@ -171,14 +174,20 @@ public:
}
// Member Operators
// Edit
//- Reset to default constructed state
void reset();
//- Reset with automatic linear/tree selection
void reset(const label procID, const label numProcs);
// Member / Friend Operators
bool operator==(const commsStruct&) const;
bool operator!=(const commsStruct&) const;
// Ostream Operator
friend Ostream& operator<<(Ostream&, const commsStruct&);
};
@ -335,6 +344,9 @@ public:
return (communicator > worldComm && communicator > selfComm);
}
//- Debugging: print the communication tree
static void printCommTree(const label communicator);
// Constructors
@ -391,24 +403,21 @@ public:
//- Move construct, takes ownership
communicator(communicator&& c) : comm_(c.comm_) { c.comm_ = -1; }
//- Allocate a communicator based on given parent
//- Allocate communicator for sub-ranks on given parent
communicator
(
//! The parent communicator
const label parent,
const label parentComm,
//! The sub-ranks of parent to use (ignore negative values)
const labelUList& subRanks,
//! Call allocatePstreamCommunicator
const bool doPstream = true
//! The sub-ranks of parent to use (negative values ignored)
const labelUList& subRanks
)
:
comm_(allocateCommunicator(parent, subRanks, doPstream))
comm_(UPstream::allocateCommunicator(parentComm, subRanks))
{}
//- Free allocated communicator and group
~communicator() { freeCommunicator(comm_); }
~communicator() { UPstream::freeCommunicator(comm_); }
//- True if communicator is non-negative (ie, was allocated)
bool good() const noexcept { return (comm_ >= 0); }
@ -417,19 +426,19 @@ public:
label comm() const noexcept { return comm_; }
//- Free allocated communicator and group
void reset() { freeCommunicator(comm_); comm_ = -1; }
void reset() { UPstream::freeCommunicator(comm_); comm_ = -1; }
//- Allocate with subRanks of parent communicator
void reset(label parent, const labelUList& subRanks)
{
freeCommunicator(comm_);
comm_ = allocateCommunicator(parent, subRanks);
UPstream::freeCommunicator(comm_);
comm_ = UPstream::allocateCommunicator(parent, subRanks);
}
//- Take ownership, free allocated communicator and group.
void reset(communicator&& c)
{
if (comm_ != c.comm_) freeCommunicator(comm_);
if (comm_ != c.comm_) UPstream::freeCommunicator(comm_);
comm_ = c.comm_;
c.comm_ = -1;
}
@ -730,22 +739,17 @@ public:
}
//- Communication schedule for linear all-to-master (proc 0)
static const List<commsStruct>& linearCommunication
static const List<commsStruct>&
linearCommunication
(
const label communicator = worldComm
)
{
return linearCommunication_[communicator];
}
);
//- Communication schedule for tree all-to-master (proc 0)
static const List<commsStruct>& treeCommunication
(
const label communicator = worldComm
)
{
return treeCommunication_[communicator];
}
);
//- Communication schedule for linear/tree all-to-master (proc 0).
//- Chooses based on the value of UPstream::nProcsSimpleSum
@ -757,8 +761,8 @@ public:
return
(
nProcs(communicator) < nProcsSimpleSum
? linearCommunication_[communicator]
: treeCommunication_[communicator]
? linearCommunication(communicator)
: treeCommunication(communicator)
);
}

252
src/OpenFOAM/db/IOstreams/Pstreams/UPstreamCommsStruct.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2016 OpenFOAM Foundation
Copyright (C) 2021-2022 OpenCFD Ltd.
Copyright (C) 2021-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -33,21 +33,21 @@ License
Foam::UPstream::commsStruct::commsStruct
(
const label above,
const labelUList& below,
const labelUList& allBelow,
const labelUList& allNotBelow
labelList&& below,
labelList&& allBelow,
labelList&& allNotBelow
)
:
above_(above),
below_(below),
allBelow_(allBelow),
allNotBelow_(allNotBelow)
below_(std::move(below)),
allBelow_(std::move(allBelow)),
allNotBelow_(std::move(allNotBelow))
{}
Foam::UPstream::commsStruct::commsStruct
(
const label nProcs,
const label numProcs,
const label myProcID,
const label above,
const labelUList& below,
@ -57,33 +57,114 @@ Foam::UPstream::commsStruct::commsStruct
above_(above),
below_(below),
allBelow_(allBelow),
allNotBelow_(nProcs - allBelow.size() - 1)
allNotBelow_(numProcs - allBelow.size() - 1)
{
boolList inBelow(nProcs, false);
List<bool> isNotBelow(numProcs, true);
forAll(allBelow, belowI)
// Exclude self
isNotBelow[myProcID] = false;
// Exclude allBelow
for (const label proci : allBelow)
{
inBelow[allBelow[belowI]] = true;
isNotBelow[proci] = false;
}
label notI = 0;
forAll(inBelow, proci)
// Compacting to obtain allNotBelow_
label nNotBelow = 0;
forAll(isNotBelow, proci)
{
if ((proci != myProcID) && !inBelow[proci])
if (isNotBelow[proci])
{
allNotBelow_[notI++] = proci;
allNotBelow_[nNotBelow++] = proci;
}
}
if (notI != allNotBelow_.size())
if (nNotBelow != allNotBelow_.size())
{
FatalErrorInFunction << "problem!" << Foam::abort(FatalError);
FatalErrorInFunction
<< "Problem: " << nNotBelow << " != " << allNotBelow_.size() << nl
<< Foam::abort(FatalError);
}
}
// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * //
// This outputs as depth-first, but graphviz sorts that for us
void Foam::UPstream::commsStruct::printGraph
(
Ostream& os,
const UList<UPstream::commsStruct>& comms,
const label proci
)
{
// if (proci >= comms.size()) return; // Extreme safety!
const auto& below = comms[proci].below();
if (proci == 0)
{
os << nl << "// communication graph:" << nl;
os.beginBlock("graph");
if (below.empty())
{
// A graph with a single-node (eg, self-comm)
os << indent << proci << nl;
}
}
int pos = 0;
for (const label nbrProci : below)
{
if (pos)
{
os << " ";
}
else
{
os << indent;
}
os << proci << " -- " << nbrProci;
if (++pos >= 4) // Max 4 items per line
{
pos = 0;
os << nl;
}
}
if (pos)
{
os << nl;
}
for (const label nbrProci : below)
{
// if (proci == nbrProci) continue; // Extreme safety!
printGraph(os, comms, nbrProci);
}
if (proci == 0)
{
os.endBlock();
os << "// end graph" << nl;
}
}
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
void Foam::UPstream::commsStruct::clear()
Foam::label Foam::UPstream::commsStruct::nProcs() const
{
return (1 + allBelow_.size() + allNotBelow_.size());
}
void Foam::UPstream::commsStruct::reset()
{
above_ = -1;
below_.clear();
@ -92,6 +173,127 @@ void Foam::UPstream::commsStruct::clear()
}
void Foam::UPstream::commsStruct::reset
(
const label procID,
const label numProcs
)
{
reset();
label above(-1);
DynamicList<label> below;
DynamicList<label> allBelow;
if (numProcs < UPstream::nProcsSimpleSum)
{
// Linear schedule
if (procID == 0)
{
below = identity(numProcs-1, 1);
allBelow = below;
}
else
{
above = 0;
}
}
else
{
// Use tree like schedule. For 8 procs:
// (level 0)
// 0 receives from 1
// 2 receives from 3
// 4 receives from 5
// 6 receives from 7
// (level 1)
// 0 receives from 2
// 4 receives from 6
// (level 2)
// 0 receives from 4
//
// The sends/receives for all levels are collected per processor
// (one send per processor; multiple receives possible) creating
// a table:
//
// So per processor:
// proc receives from sends to
// ---- ------------- --------
// 0 1,2,4 -
// 1 - 0
// 2 3 0
// 3 - 2
// 4 5 0
// 5 - 4
// 6 7 4
// 7 - 6
label mod = 0;
for (label step = 1; step < numProcs; step = mod)
{
mod = step * 2;
if (procID % mod)
{
above = procID - (procID % mod);
break;
}
else
{
for
(
label j = procID + step;
j < numProcs && j < procID + mod;
j += step
)
{
below.push_back(j);
}
for
(
label j = procID + step;
j < numProcs && j < procID + mod;
j++
)
{
allBelow.push_back(j);
}
}
}
}
*this = UPstream::commsStruct(numProcs, procID, above, below, allBelow);
}
// * * * * * * * * * * * * * * * Specializations * * * * * * * * * * * * * * //
template<>
Foam::UPstream::commsStruct&
Foam::UList<Foam::UPstream::commsStruct>::operator[](const label procID)
{
auto& val = this->v_[procID]; // or this->data()[procID]
if (val.nProcs() != size())
{
// Create/update
val.reset(procID, size());
}
return val;
}
template<>
const Foam::UPstream::commsStruct&
Foam::UList<Foam::UPstream::commsStruct>::operator[](const label procID) const
{
return const_cast<UList<UPstream::commsStruct>&>(*this).operator[](procID);
}
// * * * * * * * * * * * * * * * Member Operators * * * * * * * * * * * * * //
bool Foam::UPstream::commsStruct::operator==(const commsStruct& comm) const
@ -100,8 +302,8 @@ bool Foam::UPstream::commsStruct::operator==(const commsStruct& comm) const
(
(above_ == comm.above())
&& (below_ == comm.below())
&& (allBelow_ == allBelow())
&& (allNotBelow_ == allNotBelow())
// && (allBelow_ == comm.allBelow())
// && (allNotBelow_ == comm.allNotBelow())
);
}
@ -116,10 +318,10 @@ bool Foam::UPstream::commsStruct::operator!=(const commsStruct& comm) const
Foam::Ostream& Foam::operator<<(Ostream& os, const UPstream::commsStruct& comm)
{
os << comm.above_ << token::SPACE
<< comm.below_ << token::SPACE
<< comm.allBelow_ << token::SPACE
<< comm.allNotBelow_;
os << comm.above() << nl << token::SPACE << token::SPACE;
comm.below().writeList(os) << nl << token::SPACE << token::SPACE;
comm.allBelow().writeList(os) << nl << token::SPACE << token::SPACE;
comm.allNotBelow().writeList(os);
os.check(FUNCTION_NAME);
return os;

3
src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/eagerGAMGProcAgglomeration/eagerGAMGProcAgglomeration.C
View File

@ -64,12 +64,9 @@ Foam::eagerGAMGProcAgglomeration::
~eagerGAMGProcAgglomeration()
{
forAllReverse(comms_, i)
{
if (comms_[i] != -1)
{
UPstream::freeCommunicator(comms_[i]);
}
}
}

3
src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/manualGAMGProcAgglomeration/manualGAMGProcAgglomeration.C
View File

@ -63,12 +63,9 @@ Foam::manualGAMGProcAgglomeration::
~manualGAMGProcAgglomeration()
{
forAllReverse(comms_, i)
{
if (comms_[i] != -1)
{
UPstream::freeCommunicator(comms_[i]);
}
}
}

3
src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/masterCoarsestGAMGProcAgglomeration/masterCoarsestGAMGProcAgglomeration.C
View File

@ -103,12 +103,9 @@ Foam::masterCoarsestGAMGProcAgglomeration::
~masterCoarsestGAMGProcAgglomeration()
{
forAllReverse(comms_, i)
{
if (comms_[i] != -1)
{
UPstream::freeCommunicator(comms_[i]);
}
}
}

15
src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/procFacesGAMGProcAgglomeration/procFacesGAMGProcAgglomeration.C
View File

@ -149,10 +149,10 @@ Foam::procFacesGAMGProcAgglomeration::processorAgglomeration
const lduMesh& mesh
) const
{
label singleCellMeshComm = UPstream::allocateCommunicator
UPstream::communicator singleCellMeshComm
(
mesh.comm(),
labelList(1, Zero) // only processor 0
labelList(Foam::one{}, 0) // only processor 0
);
scalarField faceWeights;
@ -160,14 +160,14 @@ Foam::procFacesGAMGProcAgglomeration::processorAgglomeration
(
singleCellMesh
(
singleCellMeshComm,
singleCellMeshComm.comm(),
mesh,
faceWeights
)
);
tmp<labelField> tfineToCoarse(new labelField(0));
labelField& fineToCoarse = tfineToCoarse.ref();
auto tfineToCoarse = tmp<labelField>::New();
auto& fineToCoarse = tfineToCoarse.ref();
if (singleCellMeshPtr)
{
@ -197,7 +197,7 @@ Foam::procFacesGAMGProcAgglomeration::processorAgglomeration
}
Pstream::broadcast(fineToCoarse, mesh.comm());
UPstream::freeCommunicator(singleCellMeshComm);
singleCellMeshComm.reset();
return tfineToCoarse;
}
@ -233,12 +233,9 @@ Foam::procFacesGAMGProcAgglomeration::procFacesGAMGProcAgglomeration
Foam::procFacesGAMGProcAgglomeration::~procFacesGAMGProcAgglomeration()
{
forAllReverse(comms_, i)
{
if (comms_[i] != -1)
{
UPstream::freeCommunicator(comms_[i]);
}
}
}

11
src/OpenFOAM/meshes/polyMesh/globalMeshData/globalMeshData.C
View File

@ -2714,13 +2714,14 @@ void Foam::globalMeshData::updateMesh()
// *** Temporary hack to avoid problems with overlapping communication
// *** between these reductions and the calculation of deltaCoeffs
//const label comm = UPstream::worldComm + 1;
const label comm = UPstream::allocateCommunicator
UPstream::communicator dupComm
(
UPstream::worldComm,
identity(UPstream::nProcs(UPstream::worldComm)),
true
identity(UPstream::nProcs(UPstream::worldComm))
);
const label comm = dupComm.comm();
const label oldWarnComm = UPstream::commWarn(comm);
@ -2760,8 +2761,8 @@ void Foam::globalMeshData::updateMesh()
);
// Restore communicator settings
UPstream::freeCommunicator(comm);
UPstream::commWarn(oldWarnComm);
dupComm.reset();
if (debug)
{

7
src/Pstream/mpi/UPstream.C
View File

@ -335,12 +335,7 @@ bool Foam::UPstream::init(int& argc, char**& argv, const bool needsThread)
// Allocate new communicator with globalComm as its parent
const label subComm =
UPstream::allocateCommunicator
(
UPstream::globalComm, // parent
subRanks,
true
);
UPstream::allocateCommunicator(UPstream::globalComm, subRanks);
// Override worldComm

5
src/meshTools/multiWorld/multiWorldConnectionsObject.C
View File

@ -157,13 +157,12 @@ Foam::label Foam::multiWorldConnections::createCommunicator(const edge& worlds)
{
if (worlds.found(worldIDs[proci]))
{
subRanks.append(proci);
subRanks.push_back(proci);
}
}
// Allocate new communicator with global world
comm =
UPstream::allocateCommunicator(UPstream::commGlobal(), subRanks, true);
comm = UPstream::allocateCommunicator(UPstream::commGlobal(), subRanks);
if (debug & 2)
{