mirror of
https://develop.openfoam.com/Development/openfoam.git
synced 2025-11-28 03:28:01 +00:00
Merge branch 'feature-alltoall' into 'develop'
Feature alltoall Added the on-the-fly creation of communication schedule See merge request !40
This commit is contained in:
Andrew Heather
@ -3,7 +3,7 @@
|
|||||||
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
|
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
|
||||||
\\ / O peration |
|
\\ / O peration |
|
||||||
\\ / A nd | Copyright (C) 2011-2016 OpenFOAM Foundation
|
\\ / A nd | Copyright (C) 2011-2016 OpenFOAM Foundation
|
||||||
\\/ M anipulation | Copyright (C) 2015 OpenCFD Ltd.
|
\\/ M anipulation | Copyright (C) 2015-2016 OpenCFD Ltd.
|
||||||
-------------------------------------------------------------------------------
|
-------------------------------------------------------------------------------
|
||||||
License
|
License
|
||||||
This file is part of OpenFOAM.
|
This file is part of OpenFOAM.
|
||||||
@ -95,157 +95,6 @@ void Foam::UPstream::setParRun(const label nProcs)
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
Foam::List<Foam::UPstream::commsStruct> Foam::UPstream::calcLinearComm
|
|
||||||
(
|
|
||||||
const label nProcs
|
|
||||||
)
|
|
||||||
{
|
|
||||||
List<commsStruct> linearCommunication(nProcs);
|
|
||||||
|
|
||||||
// Master
|
|
||||||
labelList belowIDs(nProcs - 1);
|
|
||||||
forAll(belowIDs, i)
|
|
||||||
{
|
|
||||||
belowIDs[i] = i + 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
linearCommunication[0] = commsStruct
|
|
||||||
(
|
|
||||||
nProcs,
|
|
||||||
0,
|
|
||||||
-1,
|
|
||||||
belowIDs,
|
|
||||||
labelList(0)
|
|
||||||
);
|
|
||||||
|
|
||||||
// Slaves. Have no below processors, only communicate up to master
|
|
||||||
for (label procID = 1; procID < nProcs; procID++)
|
|
||||||
{
|
|
||||||
linearCommunication[procID] = commsStruct
|
|
||||||
(
|
|
||||||
nProcs,
|
|
||||||
procID,
|
|
||||||
0,
|
|
||||||
labelList(0),
|
|
||||||
labelList(0)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
return linearCommunication;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
void Foam::UPstream::collectReceives
|
|
||||||
(
|
|
||||||
const label procID,
|
|
||||||
const List<DynamicList<label>>& receives,
|
|
||||||
DynamicList<label>& allReceives
|
|
||||||
)
|
|
||||||
{
|
|
||||||
// Append my children (and my children children etc.) to allReceives.
|
|
||||||
|
|
||||||
const DynamicList<label>& myChildren = receives[procID];
|
|
||||||
|
|
||||||
forAll(myChildren, childI)
|
|
||||||
{
|
|
||||||
allReceives.append(myChildren[childI]);
|
|
||||||
collectReceives(myChildren[childI], receives, allReceives);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
Foam::List<Foam::UPstream::commsStruct> Foam::UPstream::calcTreeComm
|
|
||||||
(
|
|
||||||
label nProcs
|
|
||||||
)
|
|
||||||
{
|
|
||||||
// Tree like schedule. For 8 procs:
|
|
||||||
// (level 0)
|
|
||||||
// 0 receives from 1
|
|
||||||
// 2 receives from 3
|
|
||||||
// 4 receives from 5
|
|
||||||
// 6 receives from 7
|
|
||||||
// (level 1)
|
|
||||||
// 0 receives from 2
|
|
||||||
// 4 receives from 6
|
|
||||||
// (level 2)
|
|
||||||
// 0 receives from 4
|
|
||||||
//
|
|
||||||
// The sends/receives for all levels are collected per processor
|
|
||||||
// (one send per processor; multiple receives possible) creating a table:
|
|
||||||
//
|
|
||||||
// So per processor:
|
|
||||||
// proc receives from sends to
|
|
||||||
// ---- ------------- --------
|
|
||||||
// 0 1,2,4 -
|
|
||||||
// 1 - 0
|
|
||||||
// 2 3 0
|
|
||||||
// 3 - 2
|
|
||||||
// 4 5 0
|
|
||||||
// 5 - 4
|
|
||||||
// 6 7 4
|
|
||||||
// 7 - 6
|
|
||||||
|
|
||||||
label nLevels = 1;
|
|
||||||
while ((1 << nLevels) < nProcs)
|
|
||||||
{
|
|
||||||
nLevels++;
|
|
||||||
}
|
|
||||||
|
|
||||||
List<DynamicList<label>> receives(nProcs);
|
|
||||||
labelList sends(nProcs, -1);
|
|
||||||
|
|
||||||
// Info<< "Using " << nLevels << " communication levels" << endl;
|
|
||||||
|
|
||||||
label offset = 2;
|
|
||||||
label childOffset = offset/2;
|
|
||||||
|
|
||||||
for (label level = 0; level < nLevels; level++)
|
|
||||||
{
|
|
||||||
label receiveID = 0;
|
|
||||||
while (receiveID < nProcs)
|
|
||||||
{
|
|
||||||
// Determine processor that sends and we receive from
|
|
||||||
label sendID = receiveID + childOffset;
|
|
||||||
|
|
||||||
if (sendID < nProcs)
|
|
||||||
{
|
|
||||||
receives[receiveID].append(sendID);
|
|
||||||
sends[sendID] = receiveID;
|
|
||||||
}
|
|
||||||
|
|
||||||
receiveID += offset;
|
|
||||||
}
|
|
||||||
|
|
||||||
offset <<= 1;
|
|
||||||
childOffset <<= 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
// For all processors find the processors it receives data from
|
|
||||||
// (and the processors they receive data from etc.)
|
|
||||||
List<DynamicList<label>> allReceives(nProcs);
|
|
||||||
for (label procID = 0; procID < nProcs; procID++)
|
|
||||||
{
|
|
||||||
collectReceives(procID, receives, allReceives[procID]);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
List<commsStruct> treeCommunication(nProcs);
|
|
||||||
|
|
||||||
for (label procID = 0; procID < nProcs; procID++)
|
|
||||||
{
|
|
||||||
treeCommunication[procID] = commsStruct
|
|
||||||
(
|
|
||||||
nProcs,
|
|
||||||
procID,
|
|
||||||
sends[procID],
|
|
||||||
receives[procID].shrink(),
|
|
||||||
allReceives[procID].shrink()
|
|
||||||
);
|
|
||||||
}
|
|
||||||
return treeCommunication;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
Foam::label Foam::UPstream::allocateCommunicator
|
Foam::label Foam::UPstream::allocateCommunicator
|
||||||
(
|
(
|
||||||
const label parentIndex,
|
const label parentIndex,
|
||||||
@ -299,9 +148,9 @@ Foam::label Foam::UPstream::allocateCommunicator
|
|||||||
}
|
}
|
||||||
parentCommunicator_[index] = parentIndex;
|
parentCommunicator_[index] = parentIndex;
|
||||||
|
|
||||||
linearCommunication_[index] = calcLinearComm(procIDs_[index].size());
|
// Size but do not fill structure - this is done on-the-fly
|
||||||
treeCommunication_[index] = calcTreeComm(procIDs_[index].size());
|
linearCommunication_[index] = List<commsStruct>(procIDs_[index].size());
|
||||||
|
treeCommunication_[index] = List<commsStruct>(procIDs_[index].size());
|
||||||
|
|
||||||
if (doPstream && parRun())
|
if (doPstream && parRun())
|
||||||
{
|
{
|
||||||
@ -397,6 +246,115 @@ Foam::label Foam::UPstream::procNo
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
template<>
|
||||||
|
Foam::UPstream::commsStruct&
|
||||||
|
Foam::UList<Foam::UPstream::commsStruct>::operator[](const label procID)
|
||||||
|
{
|
||||||
|
UPstream::commsStruct& t = v_[procID];
|
||||||
|
|
||||||
|
if (t.allBelow().size() + t.allNotBelow().size() + 1 != size())
|
||||||
|
{
|
||||||
|
// Not yet allocated
|
||||||
|
|
||||||
|
label above(-1);
|
||||||
|
labelList below(0);
|
||||||
|
labelList allBelow(0);
|
||||||
|
|
||||||
|
if (size() < UPstream::nProcsSimpleSum)
|
||||||
|
{
|
||||||
|
// Linear schedule
|
||||||
|
|
||||||
|
if (procID == 0)
|
||||||
|
{
|
||||||
|
below.setSize(size()-1);
|
||||||
|
for (label procI = 1; procI < size(); procI++)
|
||||||
|
{
|
||||||
|
below[procI-1] = procI;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
above = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// Use tree like schedule. For 8 procs:
|
||||||
|
// (level 0)
|
||||||
|
// 0 receives from 1
|
||||||
|
// 2 receives from 3
|
||||||
|
// 4 receives from 5
|
||||||
|
// 6 receives from 7
|
||||||
|
// (level 1)
|
||||||
|
// 0 receives from 2
|
||||||
|
// 4 receives from 6
|
||||||
|
// (level 2)
|
||||||
|
// 0 receives from 4
|
||||||
|
//
|
||||||
|
// The sends/receives for all levels are collected per processor
|
||||||
|
// (one send per processor; multiple receives possible) creating
|
||||||
|
// a table:
|
||||||
|
//
|
||||||
|
// So per processor:
|
||||||
|
// proc receives from sends to
|
||||||
|
// ---- ------------- --------
|
||||||
|
// 0 1,2,4 -
|
||||||
|
// 1 - 0
|
||||||
|
// 2 3 0
|
||||||
|
// 3 - 2
|
||||||
|
// 4 5 0
|
||||||
|
// 5 - 4
|
||||||
|
// 6 7 4
|
||||||
|
// 7 - 6
|
||||||
|
|
||||||
|
label mod = 0;
|
||||||
|
|
||||||
|
for (label step = 1; step < size(); step = mod)
|
||||||
|
{
|
||||||
|
mod = step * 2;
|
||||||
|
|
||||||
|
if (procID % mod)
|
||||||
|
{
|
||||||
|
above = procID - (procID % mod);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
for
|
||||||
|
(
|
||||||
|
label j = procID + step;
|
||||||
|
j < size() && j < procID + mod;
|
||||||
|
j += step
|
||||||
|
)
|
||||||
|
{
|
||||||
|
below.append(j);
|
||||||
|
}
|
||||||
|
for
|
||||||
|
(
|
||||||
|
label j = procID + step;
|
||||||
|
j < size() && j < procID + mod;
|
||||||
|
j++
|
||||||
|
)
|
||||||
|
{
|
||||||
|
allBelow.append(j);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t = UPstream::commsStruct(size(), procID, above, below, allBelow);
|
||||||
|
}
|
||||||
|
return t;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
template<>
|
||||||
|
const Foam::UPstream::commsStruct&
|
||||||
|
Foam::UList<Foam::UPstream::commsStruct>::operator[](const label procID) const
|
||||||
|
{
|
||||||
|
return const_cast<UList<UPstream::commsStruct>& >(*this).operator[](procID);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
|
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
|
||||||
|
|
||||||
bool Foam::UPstream::parRun_(false);
|
bool Foam::UPstream::parRun_(false);
|
||||||
|
|||||||
@ -3,7 +3,7 @@
|
|||||||
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
|
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
|
||||||
\\ / O peration |
|
\\ / O peration |
|
||||||
\\ / A nd | Copyright (C) 2011-2016 OpenFOAM Foundation
|
\\ / A nd | Copyright (C) 2011-2016 OpenFOAM Foundation
|
||||||
\\/ M anipulation |
|
\\/ M anipulation | Copyright 2015-2016 OpenCFD Ltd.
|
||||||
-------------------------------------------------------------------------------
|
-------------------------------------------------------------------------------
|
||||||
License
|
License
|
||||||
This file is part of OpenFOAM.
|
This file is part of OpenFOAM.
|
||||||
@ -504,6 +504,14 @@ public:
|
|||||||
|
|
||||||
Ostream& operator<<(Ostream&, const UPstream::commsStruct&);
|
Ostream& operator<<(Ostream&, const UPstream::commsStruct&);
|
||||||
|
|
||||||
|
// Template specialisation for access of commsStruct
|
||||||
|
template<>
|
||||||
|
UPstream::commsStruct&
|
||||||
|
UList<UPstream::commsStruct>::operator[](const label);
|
||||||
|
template<>
|
||||||
|
const UPstream::commsStruct&
|
||||||
|
UList<UPstream::commsStruct>::operator[](const label) const;
|
||||||
|
|
||||||
|
|
||||||
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
|
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
|
||||||
|
|
||||||
|
|||||||
@ -3,7 +3,7 @@
|
|||||||
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
|
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
|
||||||
\\ / O peration |
|
\\ / O peration |
|
||||||
\\ / A nd | Copyright (C) 2011-2016 OpenFOAM Foundation
|
\\ / A nd | Copyright (C) 2011-2016 OpenFOAM Foundation
|
||||||
\\/ M anipulation | Copyright (C) 2015 OpenCFD Ltd.
|
\\/ M anipulation |
|
||||||
-------------------------------------------------------------------------------
|
-------------------------------------------------------------------------------
|
||||||
License
|
License
|
||||||
This file is part of OpenFOAM.
|
This file is part of OpenFOAM.
|
||||||
@ -61,8 +61,6 @@ void Foam::Pstream::exchange
|
|||||||
|
|
||||||
recvBufs.setSize(sendBufs.size());
|
recvBufs.setSize(sendBufs.size());
|
||||||
|
|
||||||
recvBufs.setSize(sendBufs.size());
|
|
||||||
|
|
||||||
if (UPstream::parRun() && UPstream::nProcs(comm) > 1)
|
if (UPstream::parRun() && UPstream::nProcs(comm) > 1)
|
||||||
{
|
{
|
||||||
label startOfRequests = Pstream::nRequests();
|
label startOfRequests = Pstream::nRequests();
|
||||||
|
|||||||
Reference in New Issue
Block a user