diff --git a/applications/test/IndirectList/Test-IndirectList.C b/applications/test/IndirectList/Test-IndirectList.C index 92c7b413a2..dae630976a 100644 --- a/applications/test/IndirectList/Test-IndirectList.C +++ b/applications/test/IndirectList/Test-IndirectList.C @@ -183,9 +183,7 @@ int main(int argc, char *argv[]) Pout<<"recv: " << flatOutput(recv) << endl; } - // MPI barrier - bool barrier = true; - Pstream::broadcast(barrier); + UPstream::barrier(UPstream::worldComm); } diff --git a/applications/test/globalIndex/Test-globalIndex.C b/applications/test/globalIndex/Test-globalIndex.C index fb92fae5b4..e62dab55e5 100644 --- a/applications/test/globalIndex/Test-globalIndex.C +++ b/applications/test/globalIndex/Test-globalIndex.C @@ -204,7 +204,7 @@ int main(int argc, char *argv[]) labelPair inOut; pointField allCcs(globalNumbering.gather(mesh.cellCentres())); inOut[0] = allCcs.size(); - Pstream::broadcast(allCcs); + Pstream::broadcastList(allCcs); inOut[1] = allCcs.size(); Pout<< " " << inOut << endl; diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H b/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H index 1f49f7234c..32afe0be87 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H @@ -102,9 +102,8 @@ public: //- Broadcast buffer content to all processes in communicator. using UPstream::broadcast; - //- Broadcast content (contiguous or non-contiguous) - //- to all processes in communicator. - // For \b non-parallel : do nothing. + //- Broadcast content (contiguous or non-contiguous) to all + //- communicator ranks. Does nothing in \b non-parallel. template static void broadcast ( @@ -112,11 +111,22 @@ public: const label comm = UPstream::worldComm ); - //- Broadcast multiple items to all processes in communicator. - // For \b non-parallel : do nothing. + //- Broadcast multiple items to all communicator ranks. + //- Does nothing in \b non-parallel. template static void broadcasts(const label comm, Type& arg1, Args&&... args); + //- Broadcast list content (contiguous or non-contiguous) to all + //- communicator ranks. Does nothing in \b non-parallel. + // For contiguous list data, this avoids serialization overhead, + // but at the expense of an additional broadcast call. + template + static void broadcastList + ( + ListType& list, + const label comm = UPstream::worldComm + ); + // Gather diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBroadcast.C b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBroadcast.C index 98c2486337..1f39cca693 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBroadcast.C +++ b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBroadcast.C @@ -80,4 +80,61 @@ void Foam::Pstream::broadcasts(const label comm, Type& arg1, Args&&... args) } +template +void Foam::Pstream::broadcastList(ListType& list, const label comm) +{ + if (is_contiguous::value) + { + // List data are contiguous + // 1. broadcast the size + // 2. resize for receiver list + // 3. broadcast contiguous contents + + if (UPstream::is_parallel(comm)) + { + label len(list.size()); + + UPstream::broadcast + ( + reinterpret_cast(&len), + sizeof(label), + comm, + UPstream::masterNo() + ); + + if (UPstream::is_subrank(comm)) + { + list.resize_nocopy(len); + } + + if (len) + { + UPstream::broadcast + ( + list.data_bytes(), + list.size_bytes(), + comm, + UPstream::masterNo() + ); + } + } + } + else if (UPstream::is_parallel(comm)) + { + // List data are non-contiguous - serialize/de-serialize + + if (UPstream::master(comm)) + { + OPBstream os(UPstream::masterNo(), comm); + os << list; + } + else // UPstream::is_subrank(comm) + { + IPBstream is(UPstream::masterNo(), comm); + is >> list; + } + } +} + + // ************************************************************************* // diff --git a/src/Pstream/mpi/UPstreamWrappingTemplates.C b/src/Pstream/mpi/UPstreamWrappingTemplates.C index caba0bc64d..475c42f7e8 100644 --- a/src/Pstream/mpi/UPstreamWrappingTemplates.C +++ b/src/Pstream/mpi/UPstreamWrappingTemplates.C @@ -613,19 +613,6 @@ void Foam::PstreamDetail::allToAllConsensus int flag = 0; MPI_Status status; -#if defined(MPI_VERSION) && (MPI_VERSION >= 3) - // MPI-3 : eg, openmpi-1.7 (2013) and later - MPI_Message message; - MPI_Improbe - ( - MPI_ANY_SOURCE, - tag, - PstreamGlobals::MPICommunicators_[comm], - &flag, - &message, - &status - ); -#else MPI_Iprobe ( MPI_ANY_SOURCE, @@ -634,7 +621,6 @@ void Foam::PstreamDetail::allToAllConsensus &flag, &status ); -#endif if (flag) { @@ -654,17 +640,6 @@ void Foam::PstreamDetail::allToAllConsensus // Regular blocking receive [the data are small] -#if defined(MPI_VERSION) && (MPI_VERSION >= 3) - // MPI-3 : eg, openmpi-1.7 (2013) and later - MPI_Mrecv - ( - &recvData[proci], - count, // count=1 (see above) - datatype, - &message, - MPI_STATUS_IGNORE - ); -#else MPI_Recv ( &recvData[proci], @@ -675,7 +650,6 @@ void Foam::PstreamDetail::allToAllConsensus PstreamGlobals::MPICommunicators_[comm], MPI_STATUS_IGNORE ); -#endif } if (barrier_active)