diff --git a/applications/utilities/mesh/advanced/combinePatchFaces/combinePatchFaces.C b/applications/utilities/mesh/advanced/combinePatchFaces/combinePatchFaces.C index 66cb396c81..c9eca74a58 100644 --- a/applications/utilities/mesh/advanced/combinePatchFaces/combinePatchFaces.C +++ b/applications/utilities/mesh/advanced/combinePatchFaces/combinePatchFaces.C @@ -48,7 +48,6 @@ Description \*---------------------------------------------------------------------------*/ -#include "PstreamReduceOps.H" #include "argList.H" #include "Time.H" #include "polyTopoChange.H" @@ -62,6 +61,7 @@ Description #include "motionSmoother.H" #include "topoSet.H" #include "processorMeshes.H" +#include "PstreamReduceOps.H" using namespace Foam; diff --git a/etc/config.csh/compiler b/etc/config.csh/compiler index bc19a6ce2a..751c5bada4 100644 --- a/etc/config.csh/compiler +++ b/etc/config.csh/compiler @@ -42,15 +42,6 @@ case ThirdParty: case Gcc48: set gcc_version=gcc-4.8.5 breaksw - case Gcc45: - set gcc_version=gcc-4.5.4 - breaksw - case Gcc46: - set gcc_version=gcc-4.6.4 - breaksw - case Gcc47: - set gcc_version=gcc-4.7.4 - breaksw case Gcc49: set gcc_version=gcc-4.9.3 breaksw @@ -69,6 +60,9 @@ case ThirdParty: case Gcc61: set gcc_version=gcc-6.1.0 breaksw + case Gcc62: + set gcc_version=gcc-6.2.0 + breaksw case Clang: set clang_version=llvm-3.7.0 # set clang_version=llvm-3.8.0 diff --git a/etc/config.sh/compiler b/etc/config.sh/compiler index 5777a1fa80..de903a0767 100644 --- a/etc/config.sh/compiler +++ b/etc/config.sh/compiler @@ -53,15 +53,15 @@ ThirdParty) Gcc53) gcc_version=gcc-5.3.0 ;; - Gcc61) - gcc_version=gcc-6.1.0 - ;; Gcc54) gcc_version=gcc-5.4.0 ;; Gcc61) gcc_version=gcc-6.1.0 ;; + Gcc62) + gcc_version=gcc-6.2.0 + ;; Clang) clang_version=llvm-3.7.0 # clang_version=llvm-3.8.0 diff --git a/etc/controlDict b/etc/controlDict index 26657f8745..38ebd71b95 100644 --- a/etc/controlDict +++ b/etc/controlDict @@ -62,6 +62,15 @@ OptimisationSwitches floatTransfer 0; nProcsSimpleSum 0; + // Optional max size (bytes) for unstructured data exchanges. In some + // phases of OpenFOAM it can send over very large data chunks + // (e.g. in parallel load balancing) and some Pstream implementations have + // problems with this. Setting this variable > 0 indicates that the + // data exchange needs to be done in multiple passes, each of maxCommsSize. + // This is not switched on by default since it requires an additional + // global reduction, even if multi-pass is not needed) + maxCommsSize 0; + // Force dumping (at next timestep) upon signal (-1 to disable) writeNowSignal -1; // 10; diff --git a/etc/cshrc b/etc/cshrc index 3d52b585a4..55f2b0174a 100644 --- a/etc/cshrc +++ b/etc/cshrc @@ -3,7 +3,7 @@ # \\ / F ield | OpenFOAM: The Open Source CFD Toolbox # \\ / O peration | # \\ / A nd | Copyright (C) 2011-2016 OpenFOAM Foundation -# \\/ M anipulation | +# \\/ M anipulation | Copyright (C) 2016 OpenCFD Ltd. #------------------------------------------------------------------------------ # License # This file is part of OpenFOAM. @@ -43,7 +43,6 @@ setenv WM_PROJECT_VERSION plus # setenv FOAM_INST_DIR `lsof +p $$ |& grep -oE '/.*'$WM_PROJECT'[^/]*/etc/cshrc' | \ sed 's%/'$WM_PROJECT'[^/]*/etc/cshrc%%'` -echo $FOAM_INST_DIR # setenv FOAM_INST_DIR $HOME/$WM_PROJECT # setenv FOAM_INST_DIR ~$WM_PROJECT # setenv FOAM_INST_DIR /opt/$WM_PROJECT diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H b/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H index e2ec02e88b..2e2f015403 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H @@ -3,7 +3,7 @@ \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | \\ / A nd | Copyright (C) 2011-2016 OpenFOAM Foundation - \\/ M anipulation | + \\/ M anipulation | Copyright (C) 2016 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -55,6 +55,37 @@ class Pstream : public UPstream { + // Private Static Functions + + //- Exchange contiguous data. Sends sendData, receives into + // recvData. If block=true will wait for all transfers to finish. + // Data provided and received as container. + template + static void exchangeContainer + ( + const UList& sendBufs, + const labelUList& recvSizes, + List& recvBufs, + const int tag, + const label comm, + const bool block + ); + + //- Exchange contiguous data. Sends sendData, receives into + // recvData. If block=true will wait for all transfers to finish. + // Data provided and received as pointers. + template + static void exchangeBuf + ( + const labelUList& sendSizes, // number of T, not number of char + const UList& sendBufs, + const labelUList& recvSizes, // number of T, not number of char + List& recvBufs, + const int tag, + const label comm, + const bool block + ); + protected: @@ -63,6 +94,7 @@ protected: //- Transfer buffer DynamicList buf_; + public: // Declare name of the class and its debug switch diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H index 53c4b6f344..51fb00060b 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H @@ -3,7 +3,7 @@ \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | \\ / A nd | Copyright (C) 2011-2016 OpenFOAM Foundation - \\/ M anipulation | + \\/ M anipulation | Copyright (C) 2016 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -32,7 +32,6 @@ Description #ifndef PstreamReduceOps_H #define PstreamReduceOps_H -#include "Pstream.H" #include "ops.H" #include "vector2D.H" diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C index d8954cb1c9..77692890b3 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C +++ b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C @@ -464,4 +464,16 @@ registerOptSwitch ); +int Foam::UPstream::maxCommsSize +( + Foam::debug::optimisationSwitch("maxCommsSize", 0) +); +registerOptSwitch +( + "maxCommsSize", + int, + Foam::UPstream::maxCommsSize +); + + // ************************************************************************* // diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H index 5ecf80de63..513c9fe036 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H @@ -269,6 +269,9 @@ public: //- Number of polling cycles in processor updates static int nPollProcInterfaces; + //- Optional maximum message size (bytes) + static int maxCommsSize; + //- Default communicator (all processors) static label worldComm; diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/exchange.C b/src/OpenFOAM/db/IOstreams/Pstreams/exchange.C index 3a3436cf27..34114a29a4 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/exchange.C +++ b/src/OpenFOAM/db/IOstreams/Pstreams/exchange.C @@ -3,7 +3,7 @@ \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | \\ / A nd | Copyright (C) 2011-2016 OpenFOAM Foundation - \\/ M anipulation | + \\/ M anipulation | Copyright (C) 2016 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -28,11 +28,157 @@ Description #include "Pstream.H" #include "contiguous.H" -#include "PstreamCombineReduceOps.H" -#include "UPstream.H" +#include "PstreamReduceOps.H" // * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // +template +void Foam::Pstream::exchangeContainer +( + const UList& sendBufs, + const labelUList& recvSizes, + List& recvBufs, + const int tag, + const label comm, + const bool block +) +{ + label startOfRequests = Pstream::nRequests(); + + // Set up receives + // ~~~~~~~~~~~~~~~ + + forAll(recvSizes, proci) + { + if (proci != Pstream::myProcNo(comm) && recvSizes[proci] > 0) + { + UIPstream::read + ( + UPstream::nonBlocking, + proci, + reinterpret_cast(recvBufs[proci].begin()), + recvSizes[proci]*sizeof(T), + tag, + comm + ); + } + } + + + // Set up sends + // ~~~~~~~~~~~~ + + forAll(sendBufs, proci) + { + if (proci != Pstream::myProcNo(comm) && sendBufs[proci].size() > 0) + { + if + ( + !UOPstream::write + ( + UPstream::nonBlocking, + proci, + reinterpret_cast(sendBufs[proci].begin()), + sendBufs[proci].size()*sizeof(T), + tag, + comm + ) + ) + { + FatalErrorInFunction + << "Cannot send outgoing message. " + << "to:" << proci << " nBytes:" + << label(sendBufs[proci].size()*sizeof(T)) + << Foam::abort(FatalError); + } + } + } + + + // Wait for all to finish + // ~~~~~~~~~~~~~~~~~~~~~~ + + if (block) + { + Pstream::waitRequests(startOfRequests); + } +} + + +template +void Foam::Pstream::exchangeBuf +( + const labelUList& sendSizes, + const UList& sendBufs, + const labelUList& recvSizes, + List& recvBufs, + const int tag, + const label comm, + const bool block +) +{ + label startOfRequests = Pstream::nRequests(); + + // Set up receives + // ~~~~~~~~~~~~~~~ + + forAll(recvSizes, proci) + { + if (proci != Pstream::myProcNo(comm) && recvSizes[proci] > 0) + { + UIPstream::read + ( + UPstream::nonBlocking, + proci, + recvBufs[proci], + recvSizes[proci]*sizeof(T), + tag, + comm + ); + } + } + + + // Set up sends + // ~~~~~~~~~~~~ + + forAll(sendBufs, proci) + { + if (proci != Pstream::myProcNo(comm) && sendSizes[proci] > 0) + { + if + ( + !UOPstream::write + ( + UPstream::nonBlocking, + proci, + sendBufs[proci], + sendSizes[proci]*sizeof(T), + tag, + comm + ) + ) + { + FatalErrorInFunction + << "Cannot send outgoing message. " + << "to:" << proci << " nBytes:" + << label(sendSizes[proci]*sizeof(T)) + << Foam::abort(FatalError); + } + } + } + + + // Wait for all to finish + // ~~~~~~~~~~~~~~~~~~~~~~ + + if (block) + { + Pstream::waitRequests(startOfRequests); + } +} + + template void Foam::Pstream::exchange ( @@ -63,11 +209,7 @@ void Foam::Pstream::exchange if (UPstream::parRun() && UPstream::nProcs(comm) > 1) { - label startOfRequests = Pstream::nRequests(); - - // Set up receives - // ~~~~~~~~~~~~~~~ - + // Presize all receive buffers forAll(recvSizes, proci) { label nRecv = recvSizes[proci]; @@ -75,55 +217,121 @@ void Foam::Pstream::exchange if (proci != Pstream::myProcNo(comm) && nRecv > 0) { recvBufs[proci].setSize(nRecv); - UIPstream::read - ( - UPstream::nonBlocking, - proci, - reinterpret_cast(recvBufs[proci].begin()), - nRecv*sizeof(T), - tag, - comm - ); } } - - // Set up sends - // ~~~~~~~~~~~~ - - forAll(sendBufs, proci) + if (Pstream::maxCommsSize <= 0) { - if (proci != Pstream::myProcNo(comm) && sendBufs[proci].size() > 0) + // Do the exchanging in one go + exchangeContainer + ( + sendBufs, + recvSizes, + recvBufs, + tag, + comm, + block + ); + } + else + { + // Determine the number of chunks to send. Note that we + // only have to look at the sending data since we are + // guaranteed that some processor's sending size is some other + // processor's receive size. Also we can ignore any local comms. + + label maxNSend = 0; + forAll(sendBufs, proci) { - if - ( - !UOPstream::write - ( - UPstream::nonBlocking, - proci, - reinterpret_cast(sendBufs[proci].begin()), - sendBufs[proci].size()*sizeof(T), - tag, - comm - ) - ) + if (proci != Pstream::myProcNo(comm)) { - FatalErrorInFunction - << "Cannot send outgoing message. " - << "to:" << proci << " nBytes:" - << label(sendBufs[proci].size()*sizeof(T)) - << Foam::abort(FatalError); + maxNSend = max(maxNSend, sendBufs[proci].size()); } } - } + + const label maxNBytes = sizeof(T)*maxNSend; + + // We need to send maxNBytes bytes so the number of iterations: + // maxNBytes iterations + // --------- ---------- + // 0 0 + // 1..maxCommsSize 1 + // maxCommsSize+1..2*maxCommsSize 2 + // etc. + + label nIter; + if (maxNBytes == 0) + { + nIter = 0; + } + else + { + nIter = (maxNBytes-1)/Pstream::maxCommsSize+1; + } + reduce(nIter, maxOp