mirror of
https://develop.openfoam.com/Development/openfoam.git
synced 2025-11-28 03:28:01 +00:00
CONTRIBUTION: Parallel: optimisation of comm structures
- Enhancements provided by Y. Inoue at RIST (http://www.hpci-office.jp) - Uses on-the-fly calculation of communication structure
This commit is contained in:
mattijs
@ -3,7 +3,7 @@
|
||||
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
|
||||
\\ / O peration |
|
||||
\\ / A nd | Copyright (C) 2011-2015 OpenFOAM Foundation
|
||||
\\/ M anipulation | Copyright (C) 2015 OpenCFD Ltd.
|
||||
\\/ M anipulation | Copyright (C) 2015-2016 OpenCFD Ltd.
|
||||
-------------------------------------------------------------------------------
|
||||
License
|
||||
This file is part of OpenFOAM.
|
||||
@ -95,155 +95,6 @@ void Foam::UPstream::setParRun(const label nProcs)
|
||||
}
|
||||
|
||||
|
||||
Foam::List<Foam::UPstream::commsStruct> Foam::UPstream::calcLinearComm
|
||||
(
|
||||
const label nProcs
|
||||
)
|
||||
{
|
||||
List<commsStruct> linearCommunication(nProcs);
|
||||
|
||||
// Master
|
||||
labelList belowIDs(nProcs - 1);
|
||||
forAll(belowIDs, i)
|
||||
{
|
||||
belowIDs[i] = i + 1;
|
||||
}
|
||||
|
||||
linearCommunication[0] = commsStruct
|
||||
(
|
||||
nProcs,
|
||||
0,
|
||||
-1,
|
||||
belowIDs,
|
||||
labelList(0)
|
||||
);
|
||||
|
||||
// Slaves. Have no below processors, only communicate up to master
|
||||
for (label procID = 1; procID < nProcs; procID++)
|
||||
{
|
||||
linearCommunication[procID] = commsStruct
|
||||
(
|
||||
nProcs,
|
||||
procID,
|
||||
0,
|
||||
labelList(0),
|
||||
labelList(0)
|
||||
);
|
||||
}
|
||||
return linearCommunication;
|
||||
}
|
||||
|
||||
|
||||
// Append my children (and my children children etc.) to allReceives.
|
||||
void Foam::UPstream::collectReceives
|
||||
(
|
||||
const label procID,
|
||||
const List<DynamicList<label> >& receives,
|
||||
DynamicList<label>& allReceives
|
||||
)
|
||||
{
|
||||
const DynamicList<label>& myChildren = receives[procID];
|
||||
|
||||
forAll(myChildren, childI)
|
||||
{
|
||||
allReceives.append(myChildren[childI]);
|
||||
collectReceives(myChildren[childI], receives, allReceives);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Tree like schedule. For 8 procs:
|
||||
// (level 0)
|
||||
// 0 receives from 1
|
||||
// 2 receives from 3
|
||||
// 4 receives from 5
|
||||
// 6 receives from 7
|
||||
// (level 1)
|
||||
// 0 receives from 2
|
||||
// 4 receives from 6
|
||||
// (level 2)
|
||||
// 0 receives from 4
|
||||
//
|
||||
// The sends/receives for all levels are collected per processor (one send per
|
||||
// processor; multiple receives possible) creating a table:
|
||||
//
|
||||
// So per processor:
|
||||
// proc receives from sends to
|
||||
// ---- ------------- --------
|
||||
// 0 1,2,4 -
|
||||
// 1 - 0
|
||||
// 2 3 0
|
||||
// 3 - 2
|
||||
// 4 5 0
|
||||
// 5 - 4
|
||||
// 6 7 4
|
||||
// 7 - 6
|
||||
Foam::List<Foam::UPstream::commsStruct> Foam::UPstream::calcTreeComm
|
||||
(
|
||||
label nProcs
|
||||
)
|
||||
{
|
||||
label nLevels = 1;
|
||||
while ((1 << nLevels) < nProcs)
|
||||
{
|
||||
nLevels++;
|
||||
}
|
||||
|
||||
List<DynamicList<label> > receives(nProcs);
|
||||
labelList sends(nProcs, -1);
|
||||
|
||||
// Info<< "Using " << nLevels << " communication levels" << endl;
|
||||
|
||||
label offset = 2;
|
||||
label childOffset = offset/2;
|
||||
|
||||
for (label level = 0; level < nLevels; level++)
|
||||
{
|
||||
label receiveID = 0;
|
||||
while (receiveID < nProcs)
|
||||
{
|
||||
// Determine processor that sends and we receive from
|
||||
label sendID = receiveID + childOffset;
|
||||
|
||||
if (sendID < nProcs)
|
||||
{
|
||||
receives[receiveID].append(sendID);
|
||||
sends[sendID] = receiveID;
|
||||
}
|
||||
|
||||
receiveID += offset;
|
||||
}
|
||||
|
||||
offset <<= 1;
|
||||
childOffset <<= 1;
|
||||
}
|
||||
|
||||
// For all processors find the processors it receives data from
|
||||
// (and the processors they receive data from etc.)
|
||||
List<DynamicList<label> > allReceives(nProcs);
|
||||
for (label procID = 0; procID < nProcs; procID++)
|
||||
{
|
||||
collectReceives(procID, receives, allReceives[procID]);
|
||||
}
|
||||
|
||||
|
||||
List<commsStruct> treeCommunication(nProcs);
|
||||
|
||||
for (label procID = 0; procID < nProcs; procID++)
|
||||
{
|
||||
treeCommunication[procID] = commsStruct
|
||||
(
|
||||
nProcs,
|
||||
procID,
|
||||
sends[procID],
|
||||
receives[procID].shrink(),
|
||||
allReceives[procID].shrink()
|
||||
);
|
||||
}
|
||||
return treeCommunication;
|
||||
}
|
||||
|
||||
|
||||
Foam::label Foam::UPstream::allocateCommunicator
|
||||
(
|
||||
const label parentIndex,
|
||||
@ -297,9 +148,12 @@ Foam::label Foam::UPstream::allocateCommunicator
|
||||
}
|
||||
parentCommunicator_[index] = parentIndex;
|
||||
|
||||
linearCommunication_[index] = calcLinearComm(procIDs_[index].size());
|
||||
treeCommunication_[index] = calcTreeComm(procIDs_[index].size());
|
||||
//linearCommunication_[index] = calcLinearComm(procIDs_[index].size());
|
||||
//treeCommunication_[index] = calcTreeComm(procIDs_[index].size());
|
||||
|
||||
// Size but do not fill structure - this is done on-the-fly
|
||||
linearCommunication_[index] = List<commsStruct>(procIDs_[index].size());
|
||||
treeCommunication_[index] = List<commsStruct>(procIDs_[index].size());
|
||||
|
||||
if (doPstream && parRun())
|
||||
{
|
||||
@ -395,6 +249,115 @@ Foam::label Foam::UPstream::procNo
|
||||
}
|
||||
|
||||
|
||||
template<>
|
||||
Foam::UPstream::commsStruct&
|
||||
Foam::UList<Foam::UPstream::commsStruct>::operator[](const label procID)
|
||||
{
|
||||
UPstream::commsStruct& t = v_[procID];
|
||||
|
||||
if (t.allBelow().size() + t.allNotBelow().size() + 1 != size())
|
||||
{
|
||||
// Not yet allocated
|
||||
|
||||
label above(-1);
|
||||
labelList below(0);
|
||||
labelList allBelow(0);
|
||||
|
||||
if (size() < UPstream::nProcsSimpleSum)
|
||||
{
|
||||
// Linear schedule
|
||||
|
||||
if (procID == 0)
|
||||
{
|
||||
below.setSize(size()-1);
|
||||
for (label procI = 1; procI < size(); procI++)
|
||||
{
|
||||
below[procI-1] = procI;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
above = 0;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Use tree like schedule. For 8 procs:
|
||||
// (level 0)
|
||||
// 0 receives from 1
|
||||
// 2 receives from 3
|
||||
// 4 receives from 5
|
||||
// 6 receives from 7
|
||||
// (level 1)
|
||||
// 0 receives from 2
|
||||
// 4 receives from 6
|
||||
// (level 2)
|
||||
// 0 receives from 4
|
||||
//
|
||||
// The sends/receives for all levels are collected per processor
|
||||
// (one send per processor; multiple receives possible) creating
|
||||
// a table:
|
||||
//
|
||||
// So per processor:
|
||||
// proc receives from sends to
|
||||
// ---- ------------- --------
|
||||
// 0 1,2,4 -
|
||||
// 1 - 0
|
||||
// 2 3 0
|
||||
// 3 - 2
|
||||
// 4 5 0
|
||||
// 5 - 4
|
||||
// 6 7 4
|
||||
// 7 - 6
|
||||
|
||||
label mod = 0;
|
||||
|
||||
for (label step = 1; step < size(); step = mod)
|
||||
{
|
||||
mod = step * 2;
|
||||
|
||||
if (procID % mod)
|
||||
{
|
||||
above = procID - (procID % mod);
|
||||
break;
|
||||
}
|
||||
else
|
||||
{
|
||||
for
|
||||
(
|
||||
label j = procID + step;
|
||||
j < size() && j < procID + mod;
|
||||
j += step
|
||||
)
|
||||
{
|
||||
below.append(j);
|
||||
}
|
||||
for
|
||||
(
|
||||
label j = procID + step;
|
||||
j < size() && j < procID + mod;
|
||||
j++
|
||||
)
|
||||
{
|
||||
allBelow.append(j);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
t = UPstream::commsStruct(size(), procID, above, below, allBelow);
|
||||
}
|
||||
return t;
|
||||
}
|
||||
|
||||
|
||||
template<>
|
||||
const Foam::UPstream::commsStruct&
|
||||
Foam::UList<Foam::UPstream::commsStruct>::operator[](const label procID) const
|
||||
{
|
||||
return const_cast<UList<UPstream::commsStruct>& >(*this).operator[](procID);
|
||||
}
|
||||
|
||||
|
||||
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
|
||||
|
||||
// By default this is not a parallel run
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
|
||||
\\ / O peration |
|
||||
\\ / A nd | Copyright (C) 2011-2014 OpenFOAM Foundation
|
||||
\\/ M anipulation | Copyright 2015 OpenCFD Ltd.
|
||||
\\/ M anipulation | Copyright 2015-2016 OpenCFD Ltd.
|
||||
-------------------------------------------------------------------------------
|
||||
License
|
||||
This file is part of OpenFOAM.
|
||||
@ -495,6 +495,14 @@ public:
|
||||
|
||||
Ostream& operator<<(Ostream&, const UPstream::commsStruct&);
|
||||
|
||||
// Template specialisation for access of commsStruct
|
||||
template<>
|
||||
UPstream::commsStruct&
|
||||
UList<UPstream::commsStruct>::operator[](const label);
|
||||
template<>
|
||||
const UPstream::commsStruct&
|
||||
UList<UPstream::commsStruct>::operator[](const label) const;
|
||||
|
||||
|
||||
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
|
||||
|
||||
|
||||
Reference in New Issue
Block a user