From 18e0d7e4d6a21c42882aad507c3155ba73a0662d Mon Sep 17 00:00:00 2001 From: Mark Olesen Date: Sat, 2 Apr 2022 14:02:54 +0200 Subject: [PATCH] ENH: bundle broadcasts (#2371) - additional Pstream::broadcasts() method to serialize/deserialize multiple items. - revoke the broadcast specialisations for std::string and List(s) and use a generic broadcasting template. In most cases, the previous specialisations would have required two broadcasts: (1) for the size (2) for the contiguous content. Now favour reduced communication over potential local (intermediate) storage that would have only benefited a few select cases. ENH: refine PstreamBuffers access methods - replace 'bool hasRecvData(label)' with 'label recvDataCount(label)' to recover the number of unconsumed receive bytes from specified processor. Can use 'labelList recvDataCounts()' to recover the number of unconsumed receive bytes from all processor. - additional peekRecvData() method (for transcribing contiguous data) ENH: globalIndex whichProcID - check for isLocal first - reasonable to assume that local items are searched for more frequently, so do preliminary check for isLocal before performing a more costly binary search of globalIndex offsets ENH: masterUncollatedFileOperation - bundled scatter of status --- .../Test-parallel-broadcast.C | 2 +- .../orientFaceZone/orientFaceZone.C | 4 +- .../splitMeshRegions/splitMeshRegions.C | 17 +- .../foamRestoreFields/foamRestoreFields.C | 2 +- .../parLagrangianRedistributor.C | 4 +- .../decomposedBlockData/decomposedBlockData.C | 75 +++++---- .../db/IOstreams/IOstreams/IOstream.H | 36 ++++- src/OpenFOAM/db/IOstreams/IOstreams/Istream.H | 4 +- src/OpenFOAM/db/IOstreams/IOstreams/Ostream.H | 13 +- src/OpenFOAM/db/IOstreams/Pstreams/Pstream.C | 38 ----- src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H | 75 ++------- .../db/IOstreams/Pstreams/PstreamBroadcast.C | 100 ++++-------- .../db/IOstreams/Pstreams/PstreamBuffers.C | 150 +++++++++++++++--- .../db/IOstreams/Pstreams/PstreamBuffers.H | 68 ++++++-- .../db/IOstreams/Pstreams/PstreamExchange.C | 4 +- .../db/IOstreams/Pstreams/UOPstreamBase.C | 6 +- src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C | 37 ----- src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H | 8 - .../collatedFileOperation/OFstreamCollator.C | 3 +- .../masterUncollatedFileOperation.C | 112 ++++++++----- .../uncollatedFileOperation.C | 8 +- .../parallel/globalIndex/globalIndex.H | 5 +- .../parallel/globalIndex/globalIndexI.H | 4 +- .../globalIndex/globalIndexTemplates.C | 2 +- src/OpenFOAM/primitives/chars/char/char.H | 2 +- src/Pstream/mpi/UPstream.C | 3 +- src/dynamicMesh/fvMeshTools/fvMeshTools.C | 23 ++- .../polyTopoChange/hexRef8/hexRef8Data.C | 2 +- .../fvMesh/zoneDistribute/zoneDistribute.C | 10 +- .../fvMesh/zoneDistribute/zoneDistributeI.H | 2 +- src/lagrangian/basic/Cloud/Cloud.C | 2 +- .../RecycleInteraction/RecycleInteraction.C | 2 +- .../state/lumpedPointState.C | 12 +- .../meshRefinement/meshRefinement.C | 2 +- .../meshRefinement/meshRefinementBaffles.C | 24 +-- .../snappyHexMeshDriver/snappyRefineDriver.C | 3 +- .../mappedPolyPatch/mappedPatchBase.C | 3 +- .../inverseDistanceCellCellStencil.C | 19 +-- .../noiseModels/surfaceNoise/surfaceNoise.C | 24 ++- .../sampledSet/sampledSets/sampledSetsImpl.C | 16 +- 40 files changed, 486 insertions(+), 440 deletions(-) diff --git a/applications/test/parallel-broadcast/Test-parallel-broadcast.C b/applications/test/parallel-broadcast/Test-parallel-broadcast.C index 44948fc1f2..38a68382e3 100644 --- a/applications/test/parallel-broadcast/Test-parallel-broadcast.C +++ b/applications/test/parallel-broadcast/Test-parallel-broadcast.C @@ -118,7 +118,7 @@ int main(int argc, char *argv[]) value = args.executable(); } printPre(value); - UPstream::broadcast(value); // Low-level UPstream broadcast + Pstream::broadcast(value); // Streamed broadcast printPost(value); } diff --git a/applications/utilities/mesh/manipulation/orientFaceZone/orientFaceZone.C b/applications/utilities/mesh/manipulation/orientFaceZone/orientFaceZone.C index 4b150e8f47..d98c4f82e2 100644 --- a/applications/utilities/mesh/manipulation/orientFaceZone/orientFaceZone.C +++ b/applications/utilities/mesh/manipulation/orientFaceZone/orientFaceZone.C @@ -211,8 +211,8 @@ int main(int argc, char *argv[]) label proci = globalFaces.whichProcID(unsetFacei); label seedFacei = globalFaces.toLocal(proci, unsetFacei); - Info<< "Seeding from processor " << proci << " face " << seedFacei - << endl; + Info<< "Seeding from processor " << proci + << " face " << seedFacei << endl; if (proci == Pstream::myProcNo()) { diff --git a/applications/utilities/mesh/manipulation/splitMeshRegions/splitMeshRegions.C b/applications/utilities/mesh/manipulation/splitMeshRegions/splitMeshRegions.C index e81301ab51..fd68ceb34a 100644 --- a/applications/utilities/mesh/manipulation/splitMeshRegions/splitMeshRegions.C +++ b/applications/utilities/mesh/manipulation/splitMeshRegions/splitMeshRegions.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2011-2017 OpenFOAM Foundation - Copyright (C) 2015-2021 OpenCFD Ltd. + Copyright (C) 2015-2022 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -516,12 +516,15 @@ void getInterfaceSizes } - // Now all processor have consistent interface information - - Pstream::scatter(interfaces); - Pstream::scatter(interfaceNames); - Pstream::scatter(interfaceSizes); - Pstream::scatter(regionsToInterface); + // Consistent interface information for all processors + Pstream::broadcasts + ( + UPstream::worldComm, + interfaces, + interfaceNames, + interfaceSizes, + regionsToInterface + ); // Mark all inter-region faces. faceToInterface.setSize(mesh.nFaces(), -1); diff --git a/applications/utilities/miscellaneous/foamRestoreFields/foamRestoreFields.C b/applications/utilities/miscellaneous/foamRestoreFields/foamRestoreFields.C index 48254f28b6..9458b3c833 100644 --- a/applications/utilities/miscellaneous/foamRestoreFields/foamRestoreFields.C +++ b/applications/utilities/miscellaneous/foamRestoreFields/foamRestoreFields.C @@ -441,7 +441,7 @@ int main(int argc, char *argv[]) { files = getFiles(args.path(), timeName); } - Pstream::scatter(files); + Pstream::broadcast(files); count += restoreFields ( diff --git a/applications/utilities/parallelProcessing/redistributePar/parLagrangianRedistributor.C b/applications/utilities/parallelProcessing/redistributePar/parLagrangianRedistributor.C index b5dbf55215..d0be2c50bb 100644 --- a/applications/utilities/parallelProcessing/redistributePar/parLagrangianRedistributor.C +++ b/applications/utilities/parallelProcessing/redistributePar/parLagrangianRedistributor.C @@ -208,9 +208,9 @@ Foam::parLagrangianRedistributor::redistributeLagrangianPositions for (const int proci : pBufs.allProcs()) { //Pout<< "Receive from processor" << proci << " : " - // << pBufs.hasRecvData(proci) << endl; + // << pBufs.recvDataCount(proci) << endl; - if (pBufs.hasRecvData(proci)) + if (pBufs.recvDataCount(proci)) { UIPstream particleStream(proci, pBufs); diff --git a/src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.C b/src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.C index be7b9dbee8..31003c8293 100644 --- a/src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.C +++ b/src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.C @@ -370,12 +370,7 @@ bool Foam::decomposedBlockData::readBlocks } else { - PstreamBuffers pBufs - ( - UPstream::commsTypes::nonBlocking, - UPstream::msgType(), - comm - ); + PstreamBuffers pBufs(comm, UPstream::commsTypes::nonBlocking); if (UPstream::master(comm)) { @@ -497,12 +492,7 @@ Foam::autoPtr Foam::decomposedBlockData::readBlocks } else { - PstreamBuffers pBufs - ( - UPstream::commsTypes::nonBlocking, - UPstream::msgType(), - comm - ); + PstreamBuffers pBufs(comm, UPstream::commsTypes::nonBlocking); if (UPstream::master(comm)) { @@ -537,13 +527,15 @@ Foam::autoPtr Foam::decomposedBlockData::readBlocks Pstream::broadcast(ok, comm); - //- Set stream properties from realIsPtr on master + // Broadcast master header info, + // set stream properties from realIsPtr on master - // Scatter master header info int verValue; int fmtValue; unsigned labelWidth; unsigned scalarWidth; + word headerName(headerIO.name()); + if (UPstream::master(comm)) { verValue = realIsPtr().version().canonical(); @@ -551,23 +543,27 @@ Foam::autoPtr Foam::decomposedBlockData::readBlocks labelWidth = realIsPtr().labelByteSize(); scalarWidth = realIsPtr().scalarByteSize(); } - Pstream::scatter(verValue); //, Pstream::msgType(), comm); - Pstream::scatter(fmtValue); //, Pstream::msgType(), comm); - Pstream::scatter(labelWidth); //, Pstream::msgType(), comm); - Pstream::scatter(scalarWidth); //, Pstream::msgType(), comm); + + Pstream::broadcasts + ( + UPstream::worldComm, // Future? comm, + verValue, + fmtValue, + labelWidth, + scalarWidth, + headerName, + headerIO.headerClassName(), + headerIO.note() + // Unneeded: headerIO.instance() + // Unneeded: headerIO.local() + ); realIsPtr().version(IOstreamOption::versionNumber::canonical(verValue)); realIsPtr().format(IOstreamOption::streamFormat(fmtValue)); realIsPtr().setLabelByteSize(labelWidth); realIsPtr().setScalarByteSize(scalarWidth); - word name(headerIO.name()); - Pstream::scatter(name, Pstream::msgType(), comm); - headerIO.rename(name); - Pstream::scatter(headerIO.headerClassName(), Pstream::msgType(), comm); - Pstream::scatter(headerIO.note(), Pstream::msgType(), comm); - //Pstream::scatter(headerIO.instance(), Pstream::msgType(), comm); - //Pstream::scatter(headerIO.local(), Pstream::msgType(), comm); + headerIO.rename(headerName); return realIsPtr; } @@ -944,6 +940,8 @@ bool Foam::decomposedBlockData::writeData(Ostream& os) const int verValue; int fmtValue; + // Unneeded: word masterName(name()); + fileName masterLocation(instance()/db().dbDir()/local()); // Re-read my own data to find out the header information if (Pstream::master(comm_)) @@ -955,24 +953,23 @@ bool Foam::decomposedBlockData::writeData(Ostream& os) const fmtValue = static_cast(headerStream.format()); } - // Scatter header information - Pstream::scatter(verValue, Pstream::msgType(), comm_); - Pstream::scatter(fmtValue, Pstream::msgType(), comm_); + // Broadcast header information + Pstream::broadcasts + ( + comm_, + verValue, + fmtValue, + // Unneeded: masterName + io.headerClassName(), + io.note(), + // Unneeded: io.instance() + // Unneeded: io.local() + masterLocation + ); streamOpt.version(IOstreamOption::versionNumber::canonical(verValue)); streamOpt.format(IOstreamOption::streamFormat(fmtValue)); - //word masterName(name()); - //Pstream::scatter(masterName, Pstream::msgType(), comm_); - - Pstream::scatter(io.headerClassName(), Pstream::msgType(), comm_); - Pstream::scatter(io.note(), Pstream::msgType(), comm_); - //Pstream::scatter(io.instance(), Pstream::msgType(), comm); - //Pstream::scatter(io.local(), Pstream::msgType(), comm); - - fileName masterLocation(instance()/db().dbDir()/local()); - Pstream::scatter(masterLocation, Pstream::msgType(), comm_); - if (!Pstream::master(comm_)) { decomposedBlockData::writeHeader diff --git a/src/OpenFOAM/db/IOstreams/IOstreams/IOstream.H b/src/OpenFOAM/db/IOstreams/IOstreams/IOstream.H index 415fd7d862..7226f7bc60 100644 --- a/src/OpenFOAM/db/IOstreams/IOstreams/IOstream.H +++ b/src/OpenFOAM/db/IOstreams/IOstreams/IOstream.H @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2011-2015 OpenFOAM Foundation - Copyright (C) 2018-2021 OpenCFD Ltd. + Copyright (C) 2018-2022 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -42,8 +42,8 @@ SourceFiles \*---------------------------------------------------------------------------*/ -#ifndef IOstream_H -#define IOstream_H +#ifndef Foam_IOstream_H +#define Foam_IOstream_H #include "char.H" #include "bool.H" @@ -469,6 +469,36 @@ inline IOstream& scientific(IOstream& io) } +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +namespace Detail +{ + +//- Termination for input looping (no-op) +template inline void inputLoop(IS&) {} + +//- Termination for output looping (no-op) +template inline void outputLoop(OS&) {} + +//- Input looping. Read into first parameter and recurse. +template +inline void inputLoop(IS& is, Type& arg1, Args&&... args) +{ + is >> arg1; + Detail::inputLoop(is, std::forward(args)...); +} + +//- Output looping. Write first parameter and recurse. +template +inline void outputLoop(OS& os, const Type& arg1, Args&&... args) +{ + os << arg1; + Detail::outputLoop(os, std::forward(args)...); +} + +} // End namespace Detail + + // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // } // End namespace Foam diff --git a/src/OpenFOAM/db/IOstreams/IOstreams/Istream.H b/src/OpenFOAM/db/IOstreams/IOstreams/Istream.H index 1d68414d7c..d14ab64fb3 100644 --- a/src/OpenFOAM/db/IOstreams/IOstreams/Istream.H +++ b/src/OpenFOAM/db/IOstreams/IOstreams/Istream.H @@ -43,8 +43,8 @@ SourceFiles \*---------------------------------------------------------------------------*/ -#ifndef Istream_H -#define Istream_H +#ifndef Foam_Istream_H +#define Foam_Istream_H #include "IOstream.H" #include "token.H" diff --git a/src/OpenFOAM/db/IOstreams/IOstreams/Ostream.H b/src/OpenFOAM/db/IOstreams/IOstreams/Ostream.H index 6f0b87e627..7f76df67b3 100644 --- a/src/OpenFOAM/db/IOstreams/IOstreams/Ostream.H +++ b/src/OpenFOAM/db/IOstreams/IOstreams/Ostream.H @@ -36,8 +36,8 @@ SourceFiles \*---------------------------------------------------------------------------*/ -#ifndef Ostream_H -#define Ostream_H +#ifndef Foam_Ostream_H +#define Foam_Ostream_H #include "IOstream.H" #include "keyType.H" @@ -50,6 +50,9 @@ namespace Foam // Forward Declarations class token; +constexpr char tab = '\t'; //!< The tab \c '\\t' character(0x09) +constexpr char nl = '\n'; //!< The newline \c '\\n' character (0x0a) + /*---------------------------------------------------------------------------*\ Class Ostream Declaration \*---------------------------------------------------------------------------*/ @@ -399,12 +402,6 @@ inline Ostream& endEntry(Ostream& os) return os; } - -// Useful aliases for tab and newline characters -constexpr char tab = '\t'; -constexpr char nl = '\n'; - - // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // } // End namespace Foam diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.C b/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.C index 0042dee05a..001766d8eb 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.C +++ b/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.C @@ -6,7 +6,6 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2011 OpenFOAM Foundation - Copyright (C) 2022 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -27,7 +26,6 @@ License \*---------------------------------------------------------------------------*/ #include "Pstream.H" -#include "bitSet.H" // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * // @@ -37,40 +35,4 @@ namespace Foam } -// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * // - -void Foam::Pstream::broadcast -( - bitSet& values, - const label comm -) -{ - if (UPstream::parRun() && UPstream::nProcs(comm) > 1) - { - // Broadcast the size - label len(values.size()); - UPstream::broadcast - ( - reinterpret_cast(&len), - sizeof(label), - comm, - UPstream::masterNo() - ); - - values.resize_nocopy(len); // A no-op on master - - if (len) - { - UPstream::broadcast - ( - values.data_bytes(), - values.size_bytes(), - comm, - UPstream::masterNo() - ); - } - } -} - - // ************************************************************************* // diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H b/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H index d7da7aec0e..b0ab673da5 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H @@ -54,9 +54,6 @@ SourceFiles namespace Foam { -// Forward Declarations -class bitSet; - /*---------------------------------------------------------------------------*\ Class Pstream Declaration \*---------------------------------------------------------------------------*/ @@ -131,61 +128,21 @@ public: // Broadcast - //- Broadcast buffer or string content to all processes in communicator + //- Broadcast buffer content to all processes in communicator. using UPstream::broadcast; - //- Generic broadcast using streams to serialize/de-serialize. - // Not normally used directly. - template - static void genericBroadcast - ( - T& value, - const label comm = UPstream::worldComm - ); - - //- Generic broadcast multiple values (contiguous or non-contiguous) + //- Broadcast content (contiguous or non-contiguous) //- to all processes in communicator. - template - static void genericListBroadcast - ( - ListType& values, - const label comm = UPstream::worldComm - ); - - //- Broadcast value (contiguous or non-contiguous) - //- to all processes in communicator. - template + template static void broadcast ( - T& value, + Type& value, const label comm = UPstream::worldComm ); - //- Broadcast multiple values (contiguous or non-contiguous) - //- to all processes in communicator. - template - static void broadcast - ( - List& values, - const label comm = UPstream::worldComm - ); - - //- Broadcast multiple values (contiguous or non-contiguous) - //- to all processes in communicator. - template - static void broadcast - ( - DynamicList& values, - const label comm = UPstream::worldComm - ); - - //- Broadcast bitSet values - //- to all processes in communicator. - static void broadcast - ( - bitSet& values, - const label comm = UPstream::worldComm - ); + //- Broadcast multiple items to all processes in communicator. + template + static void broadcasts(const label comm, Type& arg1, Args&&... args); // Gather @@ -542,9 +499,9 @@ public: const label comm = UPstream::worldComm ); - //- Helper: exchange sizes of sendData. sendData is the data per - // processor (in the communicator). Returns sizes of sendData - // on the sending processor. + //- Helper: exchange sizes of sendData. + //- The sendData is the data per processor (in the communicator). + // Returns sizes of sendData on the sending processor. template static void exchangeSizes ( @@ -554,8 +511,9 @@ public: ); - //- Helper: exchange contiguous data. Sends sendData, receives into - // recvData. If block=true will wait for all transfers to finish. + //- Helper: exchange contiguous data. + //- Sends sendData, receives into recvData. + // If wait=true will wait for all transfers to finish. template static void exchange ( @@ -567,9 +525,10 @@ public: const bool wait = true //!< Wait for requests to complete ); - //- Exchange contiguous data. Sends sendData, receives into - // recvData. Determines sizes to receive. - // If block=true will wait for all transfers to finish. + //- Exchange contiguous data. + //- Sends sendData, receives into recvData. + //- Determines sizes to receive. + // If wait=true will wait for all transfers to finish. template static void exchange ( diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBroadcast.C b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBroadcast.C index f4a92b88bc..5532260412 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBroadcast.C +++ b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBroadcast.C @@ -31,91 +31,55 @@ License // * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * // -template -void Foam::Pstream::genericBroadcast(T& value, const label comm) +template +void Foam::Pstream::broadcast(Type& value, const label comm) { - // Generic: use stream interface if (UPstream::parRun() && UPstream::nProcs(comm) > 1) { - if (UPstream::master(comm)) - { - OPBstream toAll(UPstream::masterNo(), comm); - toAll << value; - } - else - { - IPBstream fromMaster(UPstream::masterNo(), comm); - fromMaster >> value; - } - } -} - - -template -void Foam::Pstream::genericListBroadcast(ListType& values, const label comm) -{ - if (!is_contiguous::value) - { - Pstream::genericBroadcast(values, comm); - } - else if (UPstream::parRun() && UPstream::nProcs(comm) > 1) - { - // Broadcast the size - label len(values.size()); - UPstream::broadcast - ( - reinterpret_cast(&len), - sizeof(label), - comm, - UPstream::masterNo() - ); - values.resize_nocopy(len); // A no-op on master - - if (len) + if (is_contiguous::value) { + // Note: contains parallel guard internally as well UPstream::broadcast ( - values.data_bytes(), - values.size_bytes(), + reinterpret_cast(&value), + sizeof(Type), comm, UPstream::masterNo() ); } + else + { + if (UPstream::master(comm)) + { + OPBstream os(UPstream::masterNo(), comm); + os << value; + } + else + { + IPBstream is(UPstream::masterNo(), comm); + is >> value; + } + } } } -template -void Foam::Pstream::broadcast(T& value, const label comm) +template +void Foam::Pstream::broadcasts(const label comm, Type& arg1, Args&&... args) { - if (!is_contiguous::value) + if (UPstream::parRun() && UPstream::nProcs(comm) > 1) { - Pstream::genericBroadcast(value, comm); + if (UPstream::master(comm)) + { + OPBstream os(UPstream::masterNo(), comm); + Detail::outputLoop(os, arg1, std::forward(args)...); + } + else + { + IPBstream is(UPstream::masterNo(), comm); + Detail::inputLoop(is, arg1, std::forward(args)...); + } } - else if (UPstream::parRun() && UPstream::nProcs(comm) > 1) - { - UPstream::broadcast - ( - reinterpret_cast(&value), - sizeof(T), - comm, - UPstream::masterNo() - ); - } -} - - -template -void Foam::Pstream::broadcast(List& values, const label comm) -{ - Pstream::genericListBroadcast(values, comm); -} - - -template -void Foam::Pstream::broadcast(DynamicList& values, const label comm) -{ - Pstream::genericListBroadcast(values, comm); } diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.C b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.C index 55d892b9c4..db03e95838 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.C +++ b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.C @@ -172,9 +172,29 @@ Foam::PstreamBuffers::PstreamBuffers commsType_(commsType), tag_(tag), comm_(comm), - sendBuf_(UPstream::nProcs(comm)), - recvBuf_(UPstream::nProcs(comm)), - recvBufPos_(UPstream::nProcs(comm), Zero) + sendBuf_(UPstream::nProcs(comm_)), + recvBuf_(UPstream::nProcs(comm_)), + recvBufPos_(UPstream::nProcs(comm_), Zero) +{} + + +Foam::PstreamBuffers::PstreamBuffers +( + const label comm, + const UPstream::commsTypes commsType, + const int tag, + IOstreamOption::streamFormat fmt +) +: + finishedSendsCalled_(false), + allowClearRecv_(true), + format_(fmt), + commsType_(commsType), + tag_(tag), + comm_(comm), + sendBuf_(UPstream::nProcs(comm_)), + recvBuf_(UPstream::nProcs(comm_)), + recvBufPos_(UPstream::nProcs(comm_), Zero) {} @@ -215,6 +235,13 @@ void Foam::PstreamBuffers::clear() } +void Foam::PstreamBuffers::clearRecv(const label proci) +{ + recvBuf_[proci].clear(); + recvBufPos_[proci] = 0; +} + + void Foam::PstreamBuffers::clearStorage() { // Could also clear out entire sendBuf_, recvBuf_ and reallocate. @@ -246,19 +273,13 @@ bool Foam::PstreamBuffers::hasSendData() const } -bool Foam::PstreamBuffers::hasSendData(const label proci) const -{ - return !sendBuf_[proci].empty(); -} - - bool Foam::PstreamBuffers::hasRecvData() const { if (finishedSendsCalled_) { - for (const DynamicList& buf : recvBuf_) + forAll(recvBufPos_, proci) { - if (!buf.empty()) + if (recvBuf_[proci].size() > recvBufPos_[proci]) { return true; } @@ -276,11 +297,22 @@ bool Foam::PstreamBuffers::hasRecvData() const } -bool Foam::PstreamBuffers::hasRecvData(const label proci) const +Foam::label Foam::PstreamBuffers::sendDataCount(const label proci) const +{ + return sendBuf_[proci].size(); +} + + +Foam::label Foam::PstreamBuffers::recvDataCount(const label proci) const { if (finishedSendsCalled_) { - return !recvBuf_[proci].empty(); + const label len(recvBuf_[proci].size() > recvBufPos_[proci]); + + if (len > 0) + { + return len; + } } #ifdef FULLDEBUG else @@ -290,7 +322,63 @@ bool Foam::PstreamBuffers::hasRecvData(const label proci) const } #endif - return false; + return 0; +} + + +Foam::labelList Foam::PstreamBuffers::recvDataCounts() const +{ + labelList counts(recvBuf_.size(), Zero); + + if (finishedSendsCalled_) + { + forAll(recvBufPos_, proci) + { + const label len(recvBuf_[proci].size() - recvBufPos_[proci]); + + if (len > 0) + { + counts[proci] = len; + } + } + } + #ifdef FULLDEBUG + else + { + FatalErrorInFunction + << "Call finishedSends first" << exit(FatalError); + } + #endif + + return counts; +} + + +const Foam::UList +Foam::PstreamBuffers::peekRecvData(const label proci) const +{ + if (finishedSendsCalled_) + { + const label len(recvBuf_[proci].size() - recvBufPos_[proci]); + + if (len > 0) + { + return UList + ( + const_cast(&recvBuf_[proci][recvBufPos_[proci]]), + len + ); + } + } + #ifdef FULLDEBUG + else + { + FatalErrorInFunction + << "Call finishedSends first" << exit(FatalError); + } + #endif + + return UList(); } @@ -386,7 +474,7 @@ bool Foam::PstreamBuffers::finishedSends // - reasonable to assume there are no self-sends on UPstream::myProcNo forAll(sendBuf_, proci) { - // ie, hasSendData(proci) + // ie, sendDataCount(proci) != 0 if (sendConnections.set(proci, !sendBuf_[proci].empty())) { // The state changed @@ -404,7 +492,7 @@ bool Foam::PstreamBuffers::finishedSends sendProcs.clear(); forAll(sendBuf_, proci) { - // ie, hasSendData(proci) + // ie, sendDataCount(proci) != 0 if (!sendBuf_[proci].empty()) { sendProcs.append(proci); @@ -417,7 +505,7 @@ bool Foam::PstreamBuffers::finishedSends recvProcs.clear(); forAll(recvBuf_, proci) { - // ie, hasRecvData(proci) + // ie, recvDataCount(proci) if (!recvBuf_[proci].empty()) { recvProcs.append(proci); @@ -470,12 +558,34 @@ void Foam::PstreamBuffers::finishedGathers // For nonBlocking mode, simply recover received sizes // from the buffers themselves. - recvSizes.resize_nocopy(recvBuf_.size()); + recvSizes = recvDataCounts(); +} - forAll(recvBuf_, proci) + +void Foam::PstreamBuffers::finishedScatters +( + labelList& recvSizes, + const bool wait +) +{ + finalExchangeGatherScatter(false, wait); + + if (commsType_ != UPstream::commsTypes::nonBlocking) { - recvSizes[proci] = recvBuf_[proci].size(); + FatalErrorInFunction + << "Obtaining sizes not supported in " + << UPstream::commsTypeNames[commsType_] << endl + << " since transfers already in progress. Use non-blocking instead." + << exit(FatalError); + + // Note: maybe possible only if using different tag from write started + // by ~UOPstream. Needs some work. } + + // For nonBlocking mode, simply recover received sizes + // from the buffers themselves. + + recvSizes = recvDataCounts(); } diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.H b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.H index 75209b559d..44bfea19c6 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.H @@ -177,7 +177,7 @@ public: // Constructors - //- Construct given comms type, communication options, IO format + //- Construct given comms type, message tag, communicator, IO format explicit PstreamBuffers ( const UPstream::commsTypes commsType, @@ -186,6 +186,15 @@ public: IOstreamOption::streamFormat fmt = IOstreamOption::BINARY ); + //- Construct given communicator, comms type, message tag, IO format + explicit PstreamBuffers + ( + const label comm, + const UPstream::commsTypes commsType, + const int tag = UPstream::msgType(), + IOstreamOption::streamFormat fmt = IOstreamOption::BINARY + ); + //- Destructor - checks that all data have been consumed ~PstreamBuffers(); @@ -251,20 +260,6 @@ public: return finishedSendsCalled_; } - //- True if any (local) send buffers have data - bool hasSendData() const; - - //- True if (local) send buffer has data on specified processor. - bool hasSendData(const label proci) const; - - //- True if any (local) recv buffers have data. - //- Must call finishedSends() or finishedNeighbourSends() first! - bool hasRecvData() const; - - //- True if (local) recv buffer has data on specified processor. - //- Must call finishedSends() or finishedNeighbourSends() first! - bool hasRecvData(const label proci) const; - //- Is clearStorage of individual receive buffer by external hooks //- allowed? (default: true) bool allowClearRecv() const noexcept @@ -272,13 +267,42 @@ public: return allowClearRecv_; } + //- True if any (local) send buffers have data + bool hasSendData() const; + + //- True if any (local) recv buffers have unconsumed data. + //- Must call finishedSends() or other finished.. method first! + bool hasRecvData() const; + + //- Number of send bytes for the specified processor. + label sendDataCount(const label proci) const; + + //- Number of unconsumed receive bytes for the specified processor. + //- Must call finishedSends() or other finished.. method first! + label recvDataCount(const label proci) const; + + //- Number of unconsumed receive bytes for all processors. + //- Must call finishedSends() or other finished.. method first! + labelList recvDataCounts() const; + + //- Number of unconsumed receive bytes for the specified processor. + //- Must call finishedSends() or other finished.. method first! + // The method is only useful in limited situations, such as when + // PstreamBuffers has been used to fill contiguous data + // (eg, using OPstream::write). + const UList peekRecvData(const label proci) const; + // Edit //- Clear individual buffers and reset states. - // Does not clear individual buffer storage. + // Does not remove the buffer storage. void clear(); + //- Clear an individual receive buffer (eg, data not required) + // Does not remove the buffer storage. + void clearRecv(const label proci); + //- Clear individual buffer storage and reset states. void clearStorage(); @@ -406,6 +430,7 @@ public: //- Mark all sends to master as done. // // Non-blocking mode: populates receive buffers. + // Can use recvDataCounts() method to recover sizes received. // // \param wait wait for requests to complete (in nonBlocking mode) // @@ -425,11 +450,22 @@ public: //- Mark all sends to sub-procs as done. // // Non-blocking mode: populates receive buffers. + // Can use recvDataCounts() method to recover sizes received. // // \param wait wait for requests to complete (in nonBlocking mode) // // \warning currently only valid for nonBlocking comms. void finishedScatters(const bool wait = true); + + //- Mark all sends to sub-procs as done. + //- Recovers the sizes (bytes) received. + // + // Non-blocking mode: populates receive buffers (all-to-one). + // \param[out] recvSizes the sizes (bytes) received + // \param wait wait for requests to complete (in nonBlocking mode) + // + // \warning currently only valid for nonBlocking comms. + void finishedScatters(labelList& recvSizes, const bool wait = true); }; diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamExchange.C b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamExchange.C index 7ffb09e3d1..26c592c6f5 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamExchange.C +++ b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamExchange.C @@ -46,7 +46,7 @@ void Foam::Pstream::exchangeContainer const bool wait ) { - const label startOfRequests = Pstream::nRequests(); + const label startOfRequests = UPstream::nRequests(); // Set up receives // ~~~~~~~~~~~~~~~ @@ -120,7 +120,7 @@ void Foam::Pstream::exchangeBuf const bool wait ) { - const label startOfRequests = Pstream::nRequests(); + const label startOfRequests = UPstream::nRequests(); // Set up receives // ~~~~~~~~~~~~~~~ diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UOPstreamBase.C b/src/OpenFOAM/db/IOstreams/Pstreams/UOPstreamBase.C index 9cafc25cbf..7dee1199c9 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/UOPstreamBase.C +++ b/src/OpenFOAM/db/IOstreams/Pstreams/UOPstreamBase.C @@ -320,7 +320,11 @@ Foam::Ostream& Foam::UOPstreamBase::write(const doubleScalar val) } -Foam::Ostream& Foam::UOPstreamBase::write(const char* data, std::streamsize count) +Foam::Ostream& Foam::UOPstreamBase::write +( + const char* data, + std::streamsize count +) { if (format() != BINARY) { diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C index c58d61992f..37ec9d5740 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C +++ b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C @@ -51,43 +51,6 @@ Foam::UPstream::commsTypeNames }); -// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * // - -void Foam::UPstream::broadcast -( - std::string& str, - const label comm, - const int rootProcNo -) -{ - if (UPstream::parRun() && UPstream::nProcs(comm) > 1) - { - // Broadcast the string length - std::size_t len(str.length()); - - UPstream::broadcast - ( - reinterpret_cast(&len), - sizeof(std::size_t), - comm, - rootProcNo - ); - - if (!UPstream::master(comm)) - { - // Do not touch string on the master even although it would - // be a no-op. We are truly paranoid. - str.resize(len); - } - - if (len) - { - UPstream::broadcast(&str[0], len, comm, rootProcNo); - } - } -} - - // * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * // void Foam::UPstream::setParRun(const label nProcs, const bool haveThreads) diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H index 895ba8e157..2364716736 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H @@ -708,14 +708,6 @@ public: const int rootProcNo = masterNo() ); - //- Broadcast string content to all processes in communicator. - static void broadcast - ( - std::string& str, - const label communicator = worldComm, - const int rootProcNo = masterNo() - ); - // Housekeeping diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.C b/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.C index c802283c5b..d325e127a1 100644 --- a/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.C +++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.C @@ -366,8 +366,7 @@ bool Foam::OFstreamCollator::write totalSize += recvSize; maxLocalSize = max(maxLocalSize, recvSize); } - Pstream::broadcast(totalSize, localComm_); - Pstream::broadcast(maxLocalSize, localComm_); + Pstream::broadcasts(localComm_, totalSize, maxLocalSize); } if (!useThread || maxBufferSize_ == 0 || maxLocalSize > maxBufferSize_) diff --git a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C index 2f3f7cfd6e..e985abd4a0 100644 --- a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C +++ b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C @@ -578,12 +578,7 @@ Foam::fileOperations::masterUncollatedFileOperation::read // const bool uniform = uniformFile(filePaths); - PstreamBuffers pBufs - ( - Pstream::commsTypes::nonBlocking, - Pstream::msgType(), - comm - ); + PstreamBuffers pBufs(comm, UPstream::commsTypes::nonBlocking); if (Pstream::master(comm)) { @@ -1175,10 +1170,9 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::filePath // and instance have to be same { int masterType(searchType); - Pstream::broadcast(masterType); + Pstream::broadcasts(UPstream::worldComm, masterType, newInstancePath); searchType = pathType(masterType); } - UPstream::broadcast(newInstancePath); if ( @@ -1191,12 +1185,11 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::filePath { // Distribute master path. This makes sure it is seen as uniform // and only gets read from the master. - UPstream::broadcast(objPath); - UPstream::broadcast(procsDir); + Pstream::broadcasts(UPstream::worldComm, objPath, procsDir); } else { - UPstream::broadcast(procsDir, comm_); + Pstream::broadcast(procsDir, comm_); // Use the master type to determine if additional information is // needed to construct the local equivalent @@ -1276,7 +1269,7 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::dirPath // processor directory naming (void)lookupProcessorsPath(io.objectPath()); - // Determine master dirPath and scatter + // Determine master dirPath and broadcast fileName objPath; pathType searchType = NOTFOUND; @@ -1303,10 +1296,10 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::dirPath { int masterType(searchType); - Pstream::broadcast(masterType); //, comm_); + // Future?: comm_, + Pstream::broadcasts(UPstream::worldComm, masterType, newInstancePath); searchType = pathType(masterType); } - UPstream::broadcast(newInstancePath); //, comm_); if ( @@ -1319,12 +1312,11 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::dirPath { // Distribute master path. This makes sure it is seen as uniform // and only gets read from the master. - UPstream::broadcast(objPath); - UPstream::broadcast(procsDir); + Pstream::broadcasts(UPstream::worldComm, objPath, procsDir); } else { - UPstream::broadcast(procsDir, comm_); + Pstream::broadcast(procsDir, comm_); // Use the master type to determine if additional information is // needed to construct the local equivalent @@ -1479,8 +1471,8 @@ Foam::fileOperations::masterUncollatedFileOperation::findInstance } // Do parallel early exit to avoid calling time.times() - // UPstream::broadcast(foundInstance, comm_); - UPstream::broadcast(foundInstance, UPstream::worldComm); + // Pstream::broadcast(foundInstance, comm_); + Pstream::broadcast(foundInstance, UPstream::worldComm); if (!foundInstance.empty()) { io.instance() = foundInstance; @@ -1626,8 +1618,8 @@ Foam::fileOperations::masterUncollatedFileOperation::findInstance Pstream::parRun(oldParRun); } - // UPstream::broadcast(foundInstance, comm_); - UPstream::broadcast(foundInstance, UPstream::worldComm); + // Pstream::broadcast(foundInstance, comm_); + Pstream::broadcast(foundInstance, UPstream::worldComm); io.instance() = foundInstance; if (debug) { @@ -1712,8 +1704,8 @@ Foam::fileOperations::masterUncollatedFileOperation::readObjects UPstream::parRun(oldParRun); // Restore parallel state } - Pstream::broadcast(newInstance); //, comm_); - Pstream::broadcast(objectNames); //, comm_); + // Future? comm_ + Pstream::broadcasts(UPstream::worldComm, newInstance, objectNames); if (debug) { @@ -1747,7 +1739,7 @@ bool Foam::fileOperations::masterUncollatedFileOperation::readHeader filePaths[Pstream::myProcNo(Pstream::worldComm)] = fName; Pstream::gatherList(filePaths, Pstream::msgType(), Pstream::worldComm); bool uniform = uniformFile(filePaths); - Pstream::broadcast(uniform, Pstream::worldComm); + Pstream::broadcast(uniform, UPstream::worldComm); if (uniform) { @@ -1764,25 +1756,38 @@ bool Foam::fileOperations::masterUncollatedFileOperation::readHeader } } } - Pstream::broadcast(ok, Pstream::worldComm); - UPstream::broadcast(io.headerClassName(), Pstream::worldComm); - UPstream::broadcast(io.note(), Pstream::worldComm); + + Pstream::broadcasts + ( + UPstream::worldComm, + ok, + io.headerClassName(), + io.note() + ); } else { if (Pstream::nProcs(comm_) != Pstream::nProcs(Pstream::worldComm)) { // Re-gather file paths on local master - filePaths.setSize(Pstream::nProcs(comm_)); + filePaths.resize(Pstream::nProcs(comm_)); filePaths[Pstream::myProcNo(comm_)] = fName; Pstream::gatherList(filePaths, Pstream::msgType(), comm_); } - boolList result(Pstream::nProcs(comm_), false); - wordList headerClassName(Pstream::nProcs(comm_)); - stringList note(Pstream::nProcs(comm_)); + // Intermediate storage arrays (master only) + boolList result; + wordList headerClassName; + stringList note; + if (Pstream::master(comm_)) { + const label np = Pstream::nProcs(comm_); + + result.resize(np, false); + headerClassName.resize(np); + note.resize(np); + forAll(filePaths, proci) { if (!filePaths[proci].empty()) @@ -1808,14 +1813,31 @@ bool Foam::fileOperations::masterUncollatedFileOperation::readHeader } } } - ok = scatterList(result, Pstream::msgType(), comm_); - io.headerClassName() = scatterList - ( - headerClassName, - Pstream::msgType(), - comm_ - ); - io.note() = scatterList(note, Pstream::msgType(), comm_); + + // Is a more efficient scatter possible? + PstreamBuffers pBufs(comm_, UPstream::commsTypes::nonBlocking); + + if (Pstream::master(comm_)) + { + ok = result[0]; + io.headerClassName() = headerClassName[0]; + io.note() = note[0]; + + // Scatter to each proc + for (const int proci : pBufs.subProcs()) + { + UOPstream os(proci, pBufs); + os << result[proci] << headerClassName[proci] << note[proci]; + } + } + + pBufs.finishedScatters(); + + if (!Pstream::master(comm_)) + { + UIPstream is(Pstream::masterNo(), pBufs); + is >> ok >> io.headerClassName() >> io.note(); + } } if (debug) @@ -2086,9 +2108,13 @@ bool Foam::fileOperations::masterUncollatedFileOperation::read // Broadcast regIOobjects content if (Pstream::parRun()) { - Pstream::broadcast(ok, UPstream::worldComm); - UPstream::broadcast(io.headerClassName(), UPstream::worldComm); - UPstream::broadcast(io.note(), UPstream::worldComm); + Pstream::broadcasts + ( + UPstream::worldComm, + ok, + io.headerClassName(), + io.note() + ); if (Pstream::master(UPstream::worldComm)) { @@ -2513,7 +2539,7 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::getFile { fName = monitor().getFile(watchIndex); } - UPstream::broadcast(fName); //, comm_); + Pstream::broadcast(fName); //, comm_); return fName; } diff --git a/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C b/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C index d014c67e28..1f3e4eb9d6 100644 --- a/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C +++ b/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C @@ -648,8 +648,12 @@ bool Foam::fileOperations::uncollatedFileOperation::read if (masterOnly && Pstream::parRun()) { - UPstream::broadcast(io.headerClassName(), UPstream::worldComm); - UPstream::broadcast(io.note(), UPstream::worldComm); + Pstream::broadcasts + ( + UPstream::worldComm, + io.headerClassName(), + io.note() + ); if (UPstream::master(UPstream::worldComm)) { diff --git a/src/OpenFOAM/parallel/globalIndex/globalIndex.H b/src/OpenFOAM/parallel/globalIndex/globalIndex.H index 2d4496c46d..213482ee99 100644 --- a/src/OpenFOAM/parallel/globalIndex/globalIndex.H +++ b/src/OpenFOAM/parallel/globalIndex/globalIndex.H @@ -312,7 +312,10 @@ public: //- From global to local on proci inline label toLocal(const label proci, const label i) const; - //- Which processor does global come from? Binary search. + //- Which processor does global id come from? + // Does an initial check for isLocal first (assumed to occur + // reasonably frequently) followed by a binary search. + //- Fatal for out of range ids (eg, negative or >= totalSize() inline label whichProcID(const label i) const; diff --git a/src/OpenFOAM/parallel/globalIndex/globalIndexI.H b/src/OpenFOAM/parallel/globalIndex/globalIndexI.H index 4adb07cc0e..a586a3c07a 100644 --- a/src/OpenFOAM/parallel/globalIndex/globalIndexI.H +++ b/src/OpenFOAM/parallel/globalIndex/globalIndexI.H @@ -338,7 +338,9 @@ inline Foam::label Foam::globalIndex::whichProcID(const label i) const << abort(FatalError); } - return findLower(offsets_, i+1); + const label proci(Pstream::myProcNo()); + + return isLocal(proci, i) ? proci : findLower(offsets_, i+1); } diff --git a/src/OpenFOAM/parallel/globalIndex/globalIndexTemplates.C b/src/OpenFOAM/parallel/globalIndex/globalIndexTemplates.C index 1a4b59b5bc..e26a77265a 100644 --- a/src/OpenFOAM/parallel/globalIndex/globalIndexTemplates.C +++ b/src/OpenFOAM/parallel/globalIndex/globalIndexTemplates.C @@ -1042,7 +1042,7 @@ void Foam::globalIndex::get for (const int proci : sendBufs.allProcs()) { - if (sendBufs.hasRecvData(proci)) + if (sendBufs.recvDataCount(proci)) { UIPstream is(proci, sendBufs); labelList localIDs(is); diff --git a/src/OpenFOAM/primitives/chars/char/char.H b/src/OpenFOAM/primitives/chars/char/char.H index 5bd77a75bf..7708e2be9e 100644 --- a/src/OpenFOAM/primitives/chars/char/char.H +++ b/src/OpenFOAM/primitives/chars/char/char.H @@ -87,7 +87,7 @@ inline bool isspace(char c) noexcept namespace Foam { -// Template specialisation for pTraits +//- Template specialisation for pTraits\ template<> class pTraits { diff --git a/src/Pstream/mpi/UPstream.C b/src/Pstream/mpi/UPstream.C index 2e971a0201..87fb171a52 100644 --- a/src/Pstream/mpi/UPstream.C +++ b/src/Pstream/mpi/UPstream.C @@ -315,8 +315,7 @@ bool Foam::UPstream::init(int& argc, char**& argv, const bool needsThread) worldIDs_[proci] = allWorlds_.find(world); } } - Pstream::broadcast(allWorlds_); - Pstream::broadcast(worldIDs_); + Pstream::broadcasts(UPstream::worldComm, allWorlds_, worldIDs_); DynamicList