mirror of
https://develop.openfoam.com/Development/openfoam.git
synced 2025-11-28 03:28:01 +00:00
503 lines
13 KiB
C
503 lines
13 KiB
C
/*---------------------------------------------------------------------------*\
|
|
========= |
|
|
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
|
|
\\ / O peration |
|
|
\\ / A nd | Copyright (C) 2012-2013 OpenFOAM Foundation
|
|
\\/ M anipulation |
|
|
-------------------------------------------------------------------------------
|
|
License
|
|
This file is part of OpenFOAM.
|
|
|
|
OpenFOAM is free software: you can redistribute it and/or modify it
|
|
under the terms of the GNU General Public License as published by
|
|
the Free Software Foundation, either version 3 of the License, or
|
|
(at your option) any later version.
|
|
|
|
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
|
|
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
|
for more details.
|
|
|
|
You should have received a copy of the GNU General Public License
|
|
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
\*---------------------------------------------------------------------------*/
|
|
|
|
#include "UPstream.H"
|
|
#include "debug.H"
|
|
#include "dictionary.H"
|
|
#include "IOstreams.H"
|
|
|
|
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
|
|
|
|
namespace Foam
|
|
{
|
|
defineTypeNameAndDebug(UPstream, 0);
|
|
|
|
template<>
|
|
const char* Foam::NamedEnum
|
|
<
|
|
Foam::UPstream::commsTypes,
|
|
3
|
|
>::names[] =
|
|
{
|
|
"blocking",
|
|
"scheduled",
|
|
"nonBlocking"
|
|
};
|
|
}
|
|
|
|
|
|
const Foam::NamedEnum<Foam::UPstream::commsTypes, 3>
|
|
Foam::UPstream::commsTypeNames;
|
|
|
|
|
|
// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
|
|
|
|
void Foam::UPstream::setParRun(const label nProcs)
|
|
{
|
|
parRun_ = true;
|
|
|
|
// Redo worldComm communicator (this has been created at static
|
|
// initialisation time)
|
|
freeCommunicator(UPstream::worldComm);
|
|
label comm = allocateCommunicator(-1, identity(nProcs), true);
|
|
if (comm != UPstream::worldComm)
|
|
{
|
|
FatalErrorIn("UPstream::setParRun(const label)")
|
|
<< "problem : comm:" << comm
|
|
<< " UPstream::worldComm:" << UPstream::worldComm
|
|
<< Foam::exit(FatalError);
|
|
}
|
|
|
|
Pout.prefix() = '[' + name(myProcNo(Pstream::worldComm)) + "] ";
|
|
Perr.prefix() = '[' + name(myProcNo(Pstream::worldComm)) + "] ";
|
|
}
|
|
|
|
|
|
Foam::List<Foam::UPstream::commsStruct> Foam::UPstream::calcLinearComm
|
|
(
|
|
const label nProcs
|
|
)
|
|
{
|
|
List<commsStruct> linearCommunication(nProcs);
|
|
|
|
// Master
|
|
labelList belowIDs(nProcs - 1);
|
|
forAll(belowIDs, i)
|
|
{
|
|
belowIDs[i] = i + 1;
|
|
}
|
|
|
|
linearCommunication[0] = commsStruct
|
|
(
|
|
nProcs,
|
|
0,
|
|
-1,
|
|
belowIDs,
|
|
labelList(0)
|
|
);
|
|
|
|
// Slaves. Have no below processors, only communicate up to master
|
|
for (label procID = 1; procID < nProcs; procID++)
|
|
{
|
|
linearCommunication[procID] = commsStruct
|
|
(
|
|
nProcs,
|
|
procID,
|
|
0,
|
|
labelList(0),
|
|
labelList(0)
|
|
);
|
|
}
|
|
return linearCommunication;
|
|
}
|
|
|
|
|
|
// Append my children (and my children children etc.) to allReceives.
|
|
void Foam::UPstream::collectReceives
|
|
(
|
|
const label procID,
|
|
const List<DynamicList<label> >& receives,
|
|
DynamicList<label>& allReceives
|
|
)
|
|
{
|
|
const DynamicList<label>& myChildren = receives[procID];
|
|
|
|
forAll(myChildren, childI)
|
|
{
|
|
allReceives.append(myChildren[childI]);
|
|
collectReceives(myChildren[childI], receives, allReceives);
|
|
}
|
|
}
|
|
|
|
|
|
// Tree like schedule. For 8 procs:
|
|
// (level 0)
|
|
// 0 receives from 1
|
|
// 2 receives from 3
|
|
// 4 receives from 5
|
|
// 6 receives from 7
|
|
// (level 1)
|
|
// 0 receives from 2
|
|
// 4 receives from 6
|
|
// (level 2)
|
|
// 0 receives from 4
|
|
//
|
|
// The sends/receives for all levels are collected per processor (one send per
|
|
// processor; multiple receives possible) creating a table:
|
|
//
|
|
// So per processor:
|
|
// proc receives from sends to
|
|
// ---- ------------- --------
|
|
// 0 1,2,4 -
|
|
// 1 - 0
|
|
// 2 3 0
|
|
// 3 - 2
|
|
// 4 5 0
|
|
// 5 - 4
|
|
// 6 7 4
|
|
// 7 - 6
|
|
Foam::List<Foam::UPstream::commsStruct> Foam::UPstream::calcTreeComm
|
|
(
|
|
label nProcs
|
|
)
|
|
{
|
|
label nLevels = 1;
|
|
while ((1 << nLevels) < nProcs)
|
|
{
|
|
nLevels++;
|
|
}
|
|
|
|
List<DynamicList<label> > receives(nProcs);
|
|
labelList sends(nProcs, -1);
|
|
|
|
// Info<< "Using " << nLevels << " communication levels" << endl;
|
|
|
|
label offset = 2;
|
|
label childOffset = offset/2;
|
|
|
|
for (label level = 0; level < nLevels; level++)
|
|
{
|
|
label receiveID = 0;
|
|
while (receiveID < nProcs)
|
|
{
|
|
// Determine processor that sends and we receive from
|
|
label sendID = receiveID + childOffset;
|
|
|
|
if (sendID < nProcs)
|
|
{
|
|
receives[receiveID].append(sendID);
|
|
sends[sendID] = receiveID;
|
|
}
|
|
|
|
receiveID += offset;
|
|
}
|
|
|
|
offset <<= 1;
|
|
childOffset <<= 1;
|
|
}
|
|
|
|
// For all processors find the processors it receives data from
|
|
// (and the processors they receive data from etc.)
|
|
List<DynamicList<label> > allReceives(nProcs);
|
|
for (label procID = 0; procID < nProcs; procID++)
|
|
{
|
|
collectReceives(procID, receives, allReceives[procID]);
|
|
}
|
|
|
|
|
|
List<commsStruct> treeCommunication(nProcs);
|
|
|
|
for (label procID = 0; procID < nProcs; procID++)
|
|
{
|
|
treeCommunication[procID] = commsStruct
|
|
(
|
|
nProcs,
|
|
procID,
|
|
sends[procID],
|
|
receives[procID].shrink(),
|
|
allReceives[procID].shrink()
|
|
);
|
|
}
|
|
return treeCommunication;
|
|
}
|
|
|
|
|
|
//// Callback from UPstream::init() : initialize linear and tree communication
|
|
//// schedules now that nProcs is known.
|
|
//void Foam::UPstream::initCommunicationSchedule()
|
|
//{
|
|
// calcLinearComm(nProcs());
|
|
// calcTreeComm(nProcs());
|
|
//}
|
|
|
|
|
|
Foam::label Foam::UPstream::allocateCommunicator
|
|
(
|
|
const label parentIndex,
|
|
const labelList& subRanks,
|
|
const bool doPstream
|
|
)
|
|
{
|
|
label index;
|
|
if (!freeComms_.empty())
|
|
{
|
|
index = freeComms_.pop();
|
|
}
|
|
else
|
|
{
|
|
// Extend storage
|
|
index = parentCommunicator_.size();
|
|
|
|
myProcNo_.append(-1);
|
|
procIDs_.append(List<int>(0));
|
|
parentCommunicator_.append(-1);
|
|
linearCommunication_.append(List<commsStruct>(0));
|
|
treeCommunication_.append(List<commsStruct>(0));
|
|
}
|
|
|
|
Pout<< "Communicators : Allocating communicator " << index << endl
|
|
<< " parent : " << parentIndex << endl
|
|
<< " procs : " << subRanks << endl
|
|
<< endl;
|
|
|
|
// Initialise; overwritten by allocatePstreamCommunicator
|
|
myProcNo_[index] = 0;
|
|
|
|
// Convert from label to int
|
|
procIDs_[index].setSize(subRanks.size());
|
|
forAll(procIDs_[index], i)
|
|
{
|
|
procIDs_[index][i] = subRanks[i];
|
|
|
|
// Enforce incremental order (so index is rank in next communicator)
|
|
if (i >= 1 && subRanks[i] <= subRanks[i-1])
|
|
{
|
|
FatalErrorIn
|
|
(
|
|
"UPstream::allocateCommunicator"
|
|
"(const label, const labelList&, const bool)"
|
|
) << "subranks not sorted : " << subRanks
|
|
<< " when allocating subcommunicator from parent "
|
|
<< parentIndex
|
|
<< Foam::abort(FatalError);
|
|
}
|
|
}
|
|
parentCommunicator_[index] = parentIndex;
|
|
|
|
linearCommunication_[index] = calcLinearComm(procIDs_[index].size());
|
|
treeCommunication_[index] = calcTreeComm(procIDs_[index].size());
|
|
|
|
|
|
if (doPstream && parRun())
|
|
{
|
|
allocatePstreamCommunicator(parentIndex, index);
|
|
}
|
|
|
|
return index;
|
|
}
|
|
|
|
|
|
void Foam::UPstream::freeCommunicator
|
|
(
|
|
const label communicator,
|
|
const bool doPstream
|
|
)
|
|
{
|
|
Pout<< "Communicators : Freeing communicator " << communicator << endl
|
|
<< " parent : " << parentCommunicator_[communicator] << endl
|
|
<< " myProcNo : " << myProcNo_[communicator] << endl
|
|
<< endl;
|
|
|
|
if (doPstream && parRun())
|
|
{
|
|
freePstreamCommunicator(communicator);
|
|
}
|
|
myProcNo_[communicator] = -1;
|
|
//procIDs_[communicator].clear();
|
|
parentCommunicator_[communicator] = -1;
|
|
linearCommunication_[communicator].clear();
|
|
treeCommunication_[communicator].clear();
|
|
|
|
freeComms_.push(communicator);
|
|
}
|
|
|
|
|
|
void Foam::UPstream::freeCommunicators(const bool doPstream)
|
|
{
|
|
Pout<< "Communicators : Freeing all communicators" << endl
|
|
<< endl;
|
|
|
|
forAll(myProcNo_, communicator)
|
|
{
|
|
if (myProcNo_[communicator] != -1)
|
|
{
|
|
freeCommunicator(communicator, doPstream);
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
int Foam::UPstream::baseProcNo(const label myComm, const int myProcID)
|
|
{
|
|
int procID = myProcID;
|
|
label comm = myComm;
|
|
|
|
while (parent(comm) != -1)
|
|
{
|
|
const List<int>& parentRanks = UPstream::procID(comm);
|
|
procID = parentRanks[procID];
|
|
comm = UPstream::parent(comm);
|
|
}
|
|
|
|
return procID;
|
|
}
|
|
|
|
|
|
Foam::label Foam::UPstream::myProcNo(const label myComm, const int baseProcID)
|
|
{
|
|
const List<int>& parentRanks = procID(myComm);
|
|
label parentComm = parent(myComm);
|
|
|
|
if (parentComm == -1)
|
|
{
|
|
return findIndex(parentRanks, baseProcID);
|
|
}
|
|
else
|
|
{
|
|
label parentRank = myProcNo(parentComm, baseProcID);
|
|
return findIndex(parentRanks, parentRank);
|
|
}
|
|
}
|
|
|
|
|
|
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
|
|
|
|
// By default this is not a parallel run
|
|
bool Foam::UPstream::parRun_(false);
|
|
|
|
//// Initialise my process number to 0 (the master)
|
|
//int Foam::UPstream::myProcNo_(0);
|
|
//
|
|
//// List of process IDs
|
|
//Foam::List<int> Foam::UPstream::procIDs_(label(1), 0);
|
|
|
|
// Free communicators
|
|
Foam::LIFOStack<Foam::label> Foam::UPstream::freeComms_;
|
|
|
|
// My processor number
|
|
Foam::DynamicList<int> Foam::UPstream::myProcNo_(10);
|
|
|
|
// List of process IDs
|
|
Foam::DynamicList<Foam::List<int> > Foam::UPstream::procIDs_(10);
|
|
|
|
// Parent communicator
|
|
Foam::DynamicList<Foam::label> Foam::UPstream::parentCommunicator_(10);
|
|
|
|
// Standard transfer message type
|
|
int Foam::UPstream::msgType_(1);
|
|
|
|
//// Linear communication schedule
|
|
//Foam::List<Foam::UPstream::commsStruct>
|
|
// Foam::UPstream::linearCommunication_(0);
|
|
//// Multi level communication schedule
|
|
//Foam::List<Foam::UPstream::commsStruct>
|
|
// Foam::UPstream::treeCommunication_(0);
|
|
|
|
// Linear communication schedule
|
|
Foam::DynamicList<Foam::List<Foam::UPstream::commsStruct> >
|
|
Foam::UPstream::linearCommunication_(10);
|
|
|
|
// Multi level communication schedule
|
|
Foam::DynamicList<Foam::List<Foam::UPstream::commsStruct> >
|
|
Foam::UPstream::treeCommunication_(10);
|
|
|
|
|
|
// Allocate a serial communicator. This gets overwritten in parallel mode
|
|
// (by UPstream::setParRun())
|
|
Foam::UPstream::communicator serialComm(-1, Foam::labelList(1, 0), false);
|
|
|
|
|
|
|
|
// Should compact transfer be used in which floats replace doubles
|
|
// reducing the bandwidth requirement at the expense of some loss
|
|
// in accuracy
|
|
bool Foam::UPstream::floatTransfer
|
|
(
|
|
debug::optimisationSwitch("floatTransfer", 0)
|
|
);
|
|
registerOptSwitchWithName
|
|
(
|
|
Foam::UPstream::floatTransfer,
|
|
floatTransfer,
|
|
"floatTransfer"
|
|
);
|
|
|
|
// Number of processors at which the reduce algorithm changes from linear to
|
|
// tree
|
|
int Foam::UPstream::nProcsSimpleSum
|
|
(
|
|
debug::optimisationSwitch("nProcsSimpleSum", 16)
|
|
);
|
|
registerOptSwitchWithName
|
|
(
|
|
Foam::UPstream::nProcsSimpleSum,
|
|
nProcsSimpleSum,
|
|
"nProcsSimpleSum"
|
|
);
|
|
|
|
// Default commsType
|
|
Foam::UPstream::commsTypes Foam::UPstream::defaultCommsType
|
|
(
|
|
commsTypeNames.read(debug::optimisationSwitches().lookup("commsType"))
|
|
);
|
|
// Register re-reader
|
|
class addcommsTypeToOpt
|
|
:
|
|
public ::Foam::simpleRegIOobject
|
|
{
|
|
public:
|
|
addcommsTypeToOpt(const char* name)
|
|
:
|
|
::Foam::simpleRegIOobject(Foam::debug::addOptimisationObject, name)
|
|
{}
|
|
virtual ~addcommsTypeToOpt()
|
|
{}
|
|
virtual void readData(Foam::Istream& is)
|
|
{
|
|
Foam::UPstream::defaultCommsType = Foam::UPstream::commsTypeNames.read
|
|
(
|
|
is
|
|
);
|
|
}
|
|
virtual void writeData(Foam::Ostream& os) const
|
|
{
|
|
os << Foam::UPstream::commsTypeNames[Foam::UPstream::defaultCommsType];
|
|
}
|
|
};
|
|
addcommsTypeToOpt addcommsTypeToOpt_("commsType");
|
|
|
|
|
|
// Default communicator
|
|
Foam::label Foam::UPstream::worldComm(0);
|
|
|
|
|
|
// Warn for use of any communicator
|
|
Foam::label Foam::UPstream::warnComm(-1);
|
|
|
|
|
|
// Number of polling cycles in processor updates
|
|
int Foam::UPstream::nPollProcInterfaces
|
|
(
|
|
debug::optimisationSwitch("nPollProcInterfaces", 0)
|
|
);
|
|
registerOptSwitchWithName
|
|
(
|
|
Foam::UPstream::nPollProcInterfaces,
|
|
nPollProcInterfaces,
|
|
"nPollProcInterfaces"
|
|
);
|
|
|
|
// ************************************************************************* //
|