From 27dda5a092324c9cd5e55c0d3746889107b0400e Mon Sep 17 00:00:00 2001 From: mattijs Date: Mon, 4 Jan 2016 10:30:06 +0000 Subject: [PATCH 1/3] ENH: exchange: use all-to-all comms Determine sparse receive sizes instead of full matrix --- .../parLagrangianRedistributor.C | 4 +- src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H | 19 +++++-- .../db/IOstreams/Pstreams/PstreamBuffers.C | 16 ++++-- .../db/IOstreams/Pstreams/PstreamBuffers.H | 6 +- src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H | 14 ++++- src/OpenFOAM/db/IOstreams/Pstreams/exchange.C | 57 ++++++++++++++----- .../mapDistribute/mapDistributeBase.C | 4 -- src/Pstream/dummy/UPstream.C | 13 ++++- src/Pstream/mpi/UPstream.C | 37 +++++++++++- src/lagrangian/basic/Cloud/Cloud.C | 24 ++++---- src/meshTools/regionSplit/regionSplit.C | 16 +----- 11 files changed, 147 insertions(+), 63 deletions(-) diff --git a/applications/utilities/parallelProcessing/redistributePar/parLagrangianRedistributor.C b/applications/utilities/parallelProcessing/redistributePar/parLagrangianRedistributor.C index 2e775f2ef8..3cde114b3a 100644 --- a/applications/utilities/parallelProcessing/redistributePar/parLagrangianRedistributor.C +++ b/applications/utilities/parallelProcessing/redistributePar/parLagrangianRedistributor.C @@ -185,7 +185,7 @@ Foam::parLagrangianRedistributor::redistributeLagrangianPositions // Start sending. Sets number of bytes transferred - labelListList allNTrans(Pstream::nProcs()); + labelList allNTrans(Pstream::nProcs()); pBufs.finishedSends(allNTrans); @@ -208,7 +208,7 @@ Foam::parLagrangianRedistributor::redistributeLagrangianPositions // Retrieve from receive buffers forAll(allNTrans, procI) { - label nRec = allNTrans[procI][Pstream::myProcNo()]; + label nRec = allNTrans[procI]; //Pout<< "From processor " << procI << " receiving bytes " << nRec // << endl; diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H b/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H index 2a0ebd43b7..06320553e0 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H @@ -3,7 +3,7 @@ \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | \\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation - \\/ M anipulation | + \\/ M anipulation | Copyright (C) 2015 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -303,16 +303,25 @@ public: // Exchange - //- Exchange data. Sends sendData, receives into recvData, sets - // sizes (not bytes). sizes[p0][p1] is what processor p0 has - // sent to p1. Continuous data only. + //- Exchange data. Sends sendData, receives into recvData. Gets + // passed sizes to receive. Continuous data only. // If block=true will wait for all transfers to finish. + template + static void exchange + ( + const List& sendBufs, + const labelUList& recvSizes, + List& recvBufs, + const int tag = UPstream::msgType(), + const label comm = UPstream::worldComm, + const bool block = true + ); + template static void exchange ( const List&, List&, - labelListList& sizes, const int tag = UPstream::msgType(), const label comm = UPstream::worldComm, const bool block = true diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.C b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.C index f6ab958e7c..ce5d24243b 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.C +++ b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.C @@ -3,7 +3,7 @@ \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | \\ / A nd | Copyright (C) 2011-2015 OpenFOAM Foundation - \\/ M anipulation | + \\/ M anipulation | Copyright (C) 2015 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -85,12 +85,10 @@ void Foam::PstreamBuffers::finishedSends(const bool block) if (commsType_ == UPstream::nonBlocking) { - labelListList sizes; Pstream::exchange, char> ( sendBuf_, recvBuf_, - sizes, tag_, comm_, block @@ -99,17 +97,25 @@ void Foam::PstreamBuffers::finishedSends(const bool block) } -void Foam::PstreamBuffers::finishedSends(labelListList& sizes, const bool block) +void Foam::PstreamBuffers::finishedSends(labelList& recvSizes, const bool block) { finishedSendsCalled_ = true; if (commsType_ == UPstream::nonBlocking) { + labelList sendSizes(sendBuf_.size()); + forAll(sendBuf_, procI) + { + sendSizes[procI] = sendBuf_[procI].size(); + } + recvSizes.setSize(sendBuf_.size()); + UPstream::exchange(sendSizes.begin(), recvSizes.begin(), comm_); + Pstream::exchange, char> ( sendBuf_, + recvSizes, recvBuf_, - sizes, tag_, comm_, block diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.H b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.H index 718366cd5b..c8dd3d68c9 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.H @@ -3,7 +3,7 @@ \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | \\ / A nd | Copyright (C) 2011-2015 OpenFOAM Foundation - \\/ M anipulation | + \\/ M anipulation | Copyright (C) 2015 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -149,9 +149,9 @@ public: void finishedSends(const bool block = true); //- Mark all sends as having been done. Same as above but also returns - // sizes (bytes) transferred. Note:currently only valid for + // sizes (bytes) received. Note:currently only valid for // non-blocking. - void finishedSends(labelListList& sizes, const bool block = true); + void finishedSends(labelList& recvSizes, const bool block = true); //- Clear storage and reset void clear(); diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H index 2659cd33ef..4c77451d87 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H @@ -3,7 +3,7 @@ \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | \\ / A nd | Copyright (C) 2011-2014 OpenFOAM Foundation - \\/ M anipulation | + \\/ M anipulation | Copyright 2015 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -348,6 +348,18 @@ public: // Spawns slave processes and initialises inter-communication static bool init(int& argc, char**& argv); + //- Exchange single integer to/from all processors in communicator + // sendBuf, recvBuf need to be sized with number of processors in + // communicstor. Sets recvBuf to whatever sendBuf on sending processor + // is + static void exchange + ( + int* sendBuf, + int* recvBuf, + const label comm = UPstream::worldComm + ); + + // Non-blocking comms //- Get number of outstanding requests diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/exchange.C b/src/OpenFOAM/db/IOstreams/Pstreams/exchange.C index 7abcac134a..78b5ef9fbe 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/exchange.C +++ b/src/OpenFOAM/db/IOstreams/Pstreams/exchange.C @@ -37,8 +37,8 @@ template void Foam::Pstream::exchange ( const List& sendBufs, + const labelUList& recvSizes, List& recvBufs, - labelListList& sizes, const int tag, const label comm, const bool block @@ -53,23 +53,20 @@ void Foam::Pstream::exchange if (sendBufs.size() != UPstream::nProcs(comm)) { FatalErrorInFunction - << "Size of list:" << sendBufs.size() + << "Size of send buffer:" << sendBufs.size() + << " does not equal the number of processors:" + << UPstream::nProcs(comm) + << Foam::abort(FatalError); + } + if (recvSizes.size() != UPstream::nProcs(comm)) + { + FatalErrorInFunction + << "Size of receive sizes:" << recvSizes.size() << " does not equal the number of processors:" << UPstream::nProcs(comm) << Foam::abort(FatalError); } - sizes.setSize(UPstream::nProcs(comm)); - labelList& nsTransPs = sizes[UPstream::myProcNo(comm)]; - nsTransPs.setSize(UPstream::nProcs(comm)); - - forAll(sendBufs, procI) - { - nsTransPs[procI] = sendBufs[procI].size(); - } - - // Send sizes across. Note: blocks. - combineReduce(sizes, UPstream::listEq(), tag, comm); recvBufs.setSize(sendBufs.size()); @@ -80,9 +77,9 @@ void Foam::Pstream::exchange // Set up receives // ~~~~~~~~~~~~~~~ - forAll(sizes, procI) + forAll(recvSizes, procI) { - label nRecv = sizes[procI][UPstream::myProcNo(comm)]; + label nRecv = recvSizes[procI]; if (procI != Pstream::myProcNo(comm) && nRecv > 0) { @@ -144,4 +141,34 @@ void Foam::Pstream::exchange } +template +void Foam::Pstream::exchange +( + const List& sendBufs, + List& recvBufs, + const int tag, + const label comm, + const bool block +) +{ + labelList sendSizes(sendBufs.size()); + forAll(sendBufs, procI) + { + sendSizes[procI] = sendBufs[procI].size(); + } + labelList recvSizes(sendBufs.size()); + UPstream::exchange(sendSizes.begin(), recvSizes.begin(), comm); + + exchange + ( + sendBufs, + recvSizes, + recvBufs, + tag, + comm, + block + ); +} + + // ************************************************************************* // diff --git a/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeBase.C b/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeBase.C index 2cdf3d0af2..71062eee5e 100644 --- a/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeBase.C +++ b/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeBase.C @@ -446,12 +446,10 @@ void Foam::mapDistributeBase::exchangeAddressing } subMap_.setSize(Pstream::nProcs()); - labelListList sendSizes; Pstream::exchange ( wantedRemoteElements, subMap_, - sendSizes, tag, Pstream::worldComm //TBD ); @@ -526,12 +524,10 @@ void Foam::mapDistributeBase::exchangeAddressing } subMap_.setSize(Pstream::nProcs()); - labelListList sendSizes; Pstream::exchange ( wantedRemoteElements, subMap_, - sendSizes, tag, Pstream::worldComm //TBD ); diff --git a/src/Pstream/dummy/UPstream.C b/src/Pstream/dummy/UPstream.C index cd3bc60f39..75b1640c08 100644 --- a/src/Pstream/dummy/UPstream.C +++ b/src/Pstream/dummy/UPstream.C @@ -3,7 +3,7 @@ \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | \\ / A nd | Copyright (C) 2011-2015 OpenFOAM Foundation - \\/ M anipulation | + \\/ M anipulation | Copyright 2016 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -81,6 +81,17 @@ void Foam::reduce(scalar&, const sumOp&, const int, const label, label&) {} +void Foam::UPstream::exchange +( + int* sendBuf, + int* recvBuf, + const label communicator +) +{ + recvBuf[0] = sendBuf[0]; +} + + void Foam::UPstream::allocatePstreamCommunicator ( const label, diff --git a/src/Pstream/mpi/UPstream.C b/src/Pstream/mpi/UPstream.C index f035c989c3..a96968c3d6 100644 --- a/src/Pstream/mpi/UPstream.C +++ b/src/Pstream/mpi/UPstream.C @@ -3,7 +3,7 @@ \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | \\ / A nd | Copyright (C) 2011-2015 OpenFOAM Foundation - \\/ M anipulation | + \\/ M anipulation | Copyright (C) 2015 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -295,6 +295,41 @@ void Foam::reduce } +void Foam::UPstream::exchange +( + int* sendBuf, + int* recvBuf, + const label communicator +) +{ + if (!UPstream::parRun()) + { + recvBuf[0] = sendBuf[0]; + } + else + { + if + ( + MPI_Alltoall + ( + sendBuf, + 1, + MPI_INT, + recvBuf, + 1, + MPI_INT, + PstreamGlobals::MPICommunicators_[communicator] + ) + ) + { + FatalErrorInFunction + << "MPI_Alltoall failure" + << Foam::abort(FatalError); + } + } +} + + void Foam::UPstream::allocatePstreamCommunicator ( const label parentIndex, diff --git a/src/lagrangian/basic/Cloud/Cloud.C b/src/lagrangian/basic/Cloud/Cloud.C index c42136c92d..bb4598300d 100644 --- a/src/lagrangian/basic/Cloud/Cloud.C +++ b/src/lagrangian/basic/Cloud/Cloud.C @@ -3,7 +3,7 @@ \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | \\ / A nd | Copyright (C) 2011-2015 OpenFOAM Foundation - \\/ M anipulation | + \\/ M anipulation | Copyright 2015 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -321,25 +321,25 @@ void Foam::Cloud::move(TrackData& td, const scalar trackTime) } - // Start sending. Sets number of bytes transferred - labelListList allNTrans(Pstream::nProcs()); - pBufs.finishedSends(allNTrans); + // Start sending. Sets number of bytes received + labelList nRecv(Pstream::nProcs()); + pBufs.finishedSends(nRecv); bool transfered = false; - forAll(allNTrans, i) + forAll(nRecv, i) { - forAll(allNTrans[i], j) + if (nRecv[i]) { - if (allNTrans[i][j]) - { - transfered = true; - break; - } + transfered = true; + break; } } + reduce(transfered, orOp()); + + if (!transfered) { break; @@ -350,7 +350,7 @@ void Foam::Cloud::move(TrackData& td, const scalar trackTime) { label neighbProci = neighbourProcs[i]; - label nRec = allNTrans[neighbProci][Pstream::myProcNo()]; + label nRec = nRecv[neighbProci]; if (nRec) { diff --git a/src/meshTools/regionSplit/regionSplit.C b/src/meshTools/regionSplit/regionSplit.C index ea4a86551a..3540a0a4cf 100644 --- a/src/meshTools/regionSplit/regionSplit.C +++ b/src/meshTools/regionSplit/regionSplit.C @@ -344,13 +344,7 @@ Foam::autoPtr Foam::regionSplit::calcRegionSplit // Get the wanted region labels into recvNonLocal labelListList recvNonLocal(Pstream::nProcs()); - labelListList sizes; - Pstream::exchange - ( - sendNonLocal, - recvNonLocal, - sizes - ); + Pstream::exchange(sendNonLocal, recvNonLocal); // Now we have the wanted compact region labels that procI wants in // recvNonLocal[procI]. Construct corresponding list of compact @@ -372,13 +366,7 @@ Foam::autoPtr Foam::regionSplit::calcRegionSplit // Send back (into recvNonLocal) recvNonLocal.clear(); recvNonLocal.setSize(sendWantedLocal.size()); - sizes.clear(); - Pstream::exchange - ( - sendWantedLocal, - recvNonLocal, - sizes - ); + Pstream::exchange(sendWantedLocal, recvNonLocal); sendWantedLocal.clear(); // Now recvNonLocal contains for every element in setNonLocal the From 97a104311216d0cc9788b779cd9ad60dc600ff0e Mon Sep 17 00:00:00 2001 From: mattijs Date: Thu, 14 Jan 2016 09:39:29 +0000 Subject: [PATCH 2/3] CONTRIBUTION: Parallel: optimisation of all-to-all communication - Enhancements provided by Y. Inoue at RIST (http://www.hpci-office.jp) - Exchanging local sizes-to-receive only - Reversing order of receive --- .../db/IOobjects/IOdictionary/IOdictionaryIO.C | 7 ++++--- .../IOstreams/Pstreams/combineGatherScatter.C | 17 ++++++++++------- .../db/IOstreams/Pstreams/gatherScatter.C | 8 ++++++-- .../db/IOstreams/Pstreams/gatherScatterList.C | 8 ++++++-- src/OpenFOAM/db/regIOobject/regIOobjectRead.C | 5 +++-- 5 files changed, 29 insertions(+), 16 deletions(-) diff --git a/src/OpenFOAM/db/IOobjects/IOdictionary/IOdictionaryIO.C b/src/OpenFOAM/db/IOobjects/IOdictionary/IOdictionaryIO.C index d18a0e2934..8781ccc297 100644 --- a/src/OpenFOAM/db/IOobjects/IOdictionary/IOdictionaryIO.C +++ b/src/OpenFOAM/db/IOobjects/IOdictionary/IOdictionaryIO.C @@ -3,7 +3,7 @@ \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | \\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation - \\/ M anipulation | + \\/ M anipulation | Copyright 2016 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -111,8 +111,9 @@ void Foam::IOdictionary::readFile(const bool masterOnly) IOdictionary::readData(fromAbove); } - // Send to my downstairs neighbours - forAll(myComm.below(), belowI) + // Send to my downstairs neighbours. Note reverse order + // (see comment in gatherScatter.C) + forAllReverse(myComm.below(), belowI) { if (debug) { diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/combineGatherScatter.C b/src/OpenFOAM/db/IOstreams/Pstreams/combineGatherScatter.C index 1c58bd0d4d..c722741eac 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/combineGatherScatter.C +++ b/src/OpenFOAM/db/IOstreams/Pstreams/combineGatherScatter.C @@ -3,7 +3,7 @@ \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | \\ / A nd | Copyright (C) 2011-2014 OpenFOAM Foundation - \\/ M anipulation | + \\/ M anipulation | Copyright 2016 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -222,8 +222,9 @@ void Pstream::combineScatter } } - // Send to my downstairs neighbours - forAll(myComm.below(), belowI) + // Send to my downstairs neighbours. Note reverse order (see comment in + // gatherScatter.C) + forAllReverse(myComm.below(), belowI) { label belowID = myComm.below()[belowI]; @@ -461,8 +462,9 @@ void Pstream::listCombineScatter } } - // Send to my downstairs neighbours - forAll(myComm.below(), belowI) + // Send to my downstairs neighbours. Note reverse order (see comment in + // gatherScatter.C) + forAllReverse(myComm.below(), belowI) { label belowID = myComm.below()[belowI]; @@ -665,8 +667,9 @@ void Pstream::mapCombineScatter } } - // Send to my downstairs neighbours - forAll(myComm.below(), belowI) + // Send to my downstairs neighbours. Note reverse order (see comment in + // gatherScatter.C) + forAllReverse(myComm.below(), belowI) { label belowID = myComm.below()[belowI]; diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/gatherScatter.C b/src/OpenFOAM/db/IOstreams/Pstreams/gatherScatter.C index 3a250c7ab8..140d9529c0 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/gatherScatter.C +++ b/src/OpenFOAM/db/IOstreams/Pstreams/gatherScatter.C @@ -3,7 +3,7 @@ \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | \\ / A nd | Copyright (C) 2011-2014 OpenFOAM Foundation - \\/ M anipulation | + \\/ M anipulation | Copyright 2016 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -186,7 +186,11 @@ void Pstream::scatter } // Send to my downstairs neighbours - forAll(myComm.below(), belowI) + // Note that the critical path is the one last in the 'below' + // processors which is why we receive from it last and send to it + // first (to give it maximum time to digest). Note that there will + // not be any conflict since it is a directed graph. + forAllReverse(myComm.below(), belowI) { if (contiguous()) { diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/gatherScatterList.C b/src/OpenFOAM/db/IOstreams/Pstreams/gatherScatterList.C index a1330b6b6b..d7b01cb01a 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/gatherScatterList.C +++ b/src/OpenFOAM/db/IOstreams/Pstreams/gatherScatterList.C @@ -3,7 +3,7 @@ \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | \\ / A nd | Copyright (C) 2011-2015 OpenFOAM Foundation - \\/ M anipulation | Copyright (C) 2015 OpenCFD Ltd. + \\/ M anipulation | Copyright (C) 2015-2016 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -274,7 +274,11 @@ void Pstream::scatterList } // Send to my downstairs neighbours - forAll(myComm.below(), belowI) + // Note that the critical path is the one last in the 'below' + // processors which is why we receive from it last and send to it + // first (to give it maximum time to digest). Note that there will + // not be any conflict since it is a directed graph. + forAllReverse(myComm.below(), belowI) { label belowID = myComm.below()[belowI]; const labelList& notBelowLeaves = comms[belowID].allNotBelow(); diff --git a/src/OpenFOAM/db/regIOobject/regIOobjectRead.C b/src/OpenFOAM/db/regIOobject/regIOobjectRead.C index abb36aa4b1..86e9b0587c 100644 --- a/src/OpenFOAM/db/regIOobject/regIOobjectRead.C +++ b/src/OpenFOAM/db/regIOobject/regIOobjectRead.C @@ -249,8 +249,9 @@ bool Foam::regIOobject::read() ok = readData(fromAbove); } - // Send to my downstairs neighbours - forAll(myComm.below(), belowI) + // Send to my downstairs neighbours. Note reverse order (see comment in + // gatherScatter.C) + forAllReverse(myComm.below(), belowI) { OPstream toBelow ( From 3ce8fabeda24a364a7dc5edbaa45adf5842fe3af Mon Sep 17 00:00:00 2001 From: mattijs Date: Thu, 14 Jan 2016 17:36:45 +0000 Subject: [PATCH 3/3] CONTRIBUTION: Parallel: optimisation of comm structures - Enhancements provided by Y. Inoue at RIST (http://www.hpci-office.jp) - Uses on-the-fly calculation of communication structure --- src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C | 267 ++++++++---------- src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H | 10 +- 2 files changed, 124 insertions(+), 153 deletions(-) diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C index 87b21b43b3..fdfbdf882e 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C +++ b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C @@ -3,7 +3,7 @@ \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | \\ / A nd | Copyright (C) 2011-2015 OpenFOAM Foundation - \\/ M anipulation | Copyright (C) 2015 OpenCFD Ltd. + \\/ M anipulation | Copyright (C) 2015-2016 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -95,155 +95,6 @@ void Foam::UPstream::setParRun(const label nProcs) } -Foam::List Foam::UPstream::calcLinearComm -( - const label nProcs -) -{ - List linearCommunication(nProcs); - - // Master - labelList belowIDs(nProcs - 1); - forAll(belowIDs, i) - { - belowIDs[i] = i + 1; - } - - linearCommunication[0] = commsStruct - ( - nProcs, - 0, - -1, - belowIDs, - labelList(0) - ); - - // Slaves. Have no below processors, only communicate up to master - for (label procID = 1; procID < nProcs; procID++) - { - linearCommunication[procID] = commsStruct - ( - nProcs, - procID, - 0, - labelList(0), - labelList(0) - ); - } - return linearCommunication; -} - - -// Append my children (and my children children etc.) to allReceives. -void Foam::UPstream::collectReceives -( - const label procID, - const List >& receives, - DynamicList