From 5d9f8e9a9d69c2f42ec39adfac41f09aa62bf0cc Mon Sep 17 00:00:00 2001 From: Mark Olesen Date: Wed, 26 Feb 2025 17:39:23 +0100 Subject: [PATCH] ENH: remove gatherv/scatterv direct coding - includes intrinsic MPI types, but no component aggregates since we barely wish to use gatherv/scatterv (slow!) in the first place and it makes no sense to recalculate the list of counts for component aggregates which we will never use. --- src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H | 118 ++++++++---------- .../db/IOstreams/Pstreams/UPstream.txx | 101 +++++++++++++++ src/Pstream/dummy/UPstreamGatherScatter.C | 46 ------- src/Pstream/mpi/UPstreamGatherScatter.C | 55 -------- 4 files changed, 155 insertions(+), 165 deletions(-) diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H index b80eba3f34..8caf255853 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H @@ -1619,71 +1619,29 @@ public: const int communicator = UPstream::worldComm ); + //- Receive variable length data from all ranks + template + static void mpiGatherv + ( + const Type* sendData, + int sendCount, //!< Ignored on master if recvCount[0] == 0 + Type* recvData, //!< Ignored on non-root rank + const UList& recvCounts, //!< Ignored on non-root rank + const UList& recvOffsets, //!< Ignored on non-root rank + const int communicator = UPstream::worldComm + ); - #undef Pstream_CommonRoutines - #define Pstream_CommonRoutines(Type) \ - \ - /*! \brief Receive variable length \c Type data from all ranks */ \ - static void mpiGatherv \ - ( \ - const Type* sendData, \ - int sendCount, /*!< Ignored on master if recvCount[0] == 0 */ \ - Type* recvData, /*!< Ignored on non-root rank */ \ - const UList& recvCounts, /*!< Ignored on non-root rank */ \ - const UList& recvOffsets, /*!< Ignored on non-root rank */ \ - const label communicator = worldComm \ - ); \ - \ - /*! \brief Send variable length \c Type data to all ranks */ \ - static void mpiScatterv \ - ( \ - const Type* sendData, /*!< Ignored on non-root rank */ \ - const UList& sendCounts, /*!< Ignored on non-root rank */ \ - const UList& sendOffsets, /*!< Ignored on non-root rank */ \ - Type* recvData, \ - int recvCount, \ - const label communicator = worldComm \ - ); \ - \ - /*! \deprecated(2025-02) prefer mpiGatherv */ \ - FOAM_DEPRECATED_FOR(2025-02, "mpiGatherv()") \ - inline static void gather \ - ( \ - const Type* send, \ - int count, \ - Type* recv, \ - const UList& counts, \ - const UList& offsets, \ - const label comm = worldComm \ - ) \ - { \ - UPstream::mpiGatherv(send, count, recv, counts, offsets, comm); \ - } \ - \ - /*! \deprecated(2025-02) prefer mpiScatterv */ \ - FOAM_DEPRECATED_FOR(2025-02, "mpiScatterv()") \ - inline static void scatter \ - ( \ - const Type* send, \ - const UList& counts, \ - const UList& offsets, \ - Type* recv, \ - int count, \ - const label comm = worldComm \ - ) \ - { \ - UPstream::mpiScatterv(send, counts, offsets, recv, count, comm); \ - } - - Pstream_CommonRoutines(char); - Pstream_CommonRoutines(int32_t); - Pstream_CommonRoutines(int64_t); - Pstream_CommonRoutines(uint32_t); - Pstream_CommonRoutines(uint64_t); - Pstream_CommonRoutines(float); - Pstream_CommonRoutines(double); - - #undef Pstream_CommonRoutines + //- Send variable length data to all ranks + template + static void mpiScatterv + ( + const Type* sendData, //!< Ignored on non-root rank + const UList& sendCounts, //!< Ignored on non-root rank + const UList& sendOffsets, //!< Ignored on non-root rank + Type* recvData, + int recvCount, + const int communicator = UPstream::worldComm + ); // Gather single, contiguous value(s) @@ -1832,6 +1790,38 @@ public: // Should normally be restricted to a particular starting request. FOAM_DEPRECATED_FOR(2023-01, "waitRequests(int) method") static void waitRequests() { waitRequests(0); } + + //- \deprecated(2025-02) prefer mpiGatherv + template + FOAM_DEPRECATED_FOR(2025-02, "mpiGatherv()") + static void gather + ( + const Type* send, + int count, + Type* recv, + const UList& counts, + const UList& offsets, + const int comm = UPstream::worldComm + ) + { + UPstream::mpiGatherv(send, count, recv, counts, offsets, comm); + } + + //- \deprecated(2025-02) prefer mpiScatterv + template + FOAM_DEPRECATED_FOR(2025-02, "mpiScatterv()") + static void scatter + ( + const Type* send, + const UList& counts, + const UList& offsets, + Type* recv, + int count, + const int comm = UPstream::worldComm + ) + { + UPstream::mpiScatterv(send, counts, offsets, recv, count, comm); + } }; diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.txx b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.txx index 58aae735e9..1e529ac018 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.txx +++ b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.txx @@ -318,6 +318,107 @@ T Foam::UPstream::listScatterValues } +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +template +void Foam::UPstream::mpiGatherv +( + const Type* sendData, + int sendCount, + Type* recvData, + const UList& recvCounts, + const UList& recvOffsets, + const int communicator +) +{ + if (!UPstream::is_parallel(communicator)) + { + if constexpr (is_contiguous_v) + { + if (sendData && recvData) + { + // recvCounts[0] may be invalid - use sendCount instead + std::memmove(recvData, sendData, sendCount*sizeof(Type)); + } + } + // Nothing further to do + } + else if constexpr (UPstream_basic_dataType::value) + { + // Restrict to basic (or aliased) MPI types to avoid recalculating + // the list of counts/offsets. + + UPstream::mpi_gatherv + ( + sendData, + sendCount, + recvData, + recvCounts, + recvOffsets, + + UPstream_basic_dataType::datatype_id, + communicator + ); + } + else + { + static_assert + ( + stdFoam::dependent_false_v, "Only basic MPI data types" + ); + } +} + + +template +void Foam::UPstream::mpiScatterv +( + const Type* sendData, + const UList& sendCounts, + const UList& sendOffsets, + Type* recvData, + int recvCount, + const int communicator +) +{ + if (!UPstream::is_parallel(communicator)) + { + if constexpr (is_contiguous_v) + { + if (sendData && recvData) + { + std::memmove(recvData, sendData, recvCount*sizeof(Type)); + } + } + // Nothing further to do + } + else if constexpr (UPstream_basic_dataType::value) + { + // Restrict to basic (or aliased) MPI types to avoid recalculating + // the list of counts/offsets. + + UPstream::mpi_scatterv + ( + sendData, + sendCounts, + sendOffsets, + recvData, + recvCount, + + UPstream_basic_dataType::datatype_id, + communicator + ); + } + else + { + static_assert + ( + stdFoam::dependent_false_v, "Only basic MPI data types" + ); + } +} + + // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // template diff --git a/src/Pstream/dummy/UPstreamGatherScatter.C b/src/Pstream/dummy/UPstreamGatherScatter.C index e5f4fcbe84..4d4e3992ef 100644 --- a/src/Pstream/dummy/UPstreamGatherScatter.C +++ b/src/Pstream/dummy/UPstreamGatherScatter.C @@ -98,50 +98,4 @@ void Foam::UPstream::mpi_scatterv {} -// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // - -#undef Pstream_CommonRoutines -#define Pstream_CommonRoutines(Type) \ - \ -void Foam::UPstream::mpiGatherv \ -( \ - const Type* sendData, \ - int sendCount, \ - \ - Type* recvData, \ - const UList& recvCounts, \ - const UList& recvOffsets, \ - const label comm \ -) \ -{ \ - /* recvCounts[0] may be invalid - use sendCount instead */ \ - std::memmove(recvData, sendData, sendCount*sizeof(Type)); \ -} \ - \ -void Foam::UPstream::mpiScatterv \ -( \ - const Type* sendData, \ - const UList& sendCounts, \ - const UList& sendOffsets, \ - \ - Type* recvData, \ - int recvCount, \ - const label comm \ -) \ -{ \ - std::memmove(recvData, sendData, recvCount*sizeof(Type)); \ -} - - -//TDB: Pstream_CommonRoutines(bool); -Pstream_CommonRoutines(char); -Pstream_CommonRoutines(int32_t); -Pstream_CommonRoutines(int64_t); -Pstream_CommonRoutines(uint32_t); -Pstream_CommonRoutines(uint64_t); -Pstream_CommonRoutines(float); -Pstream_CommonRoutines(double); - -#undef Pstream_CommonRoutines - // ************************************************************************* // diff --git a/src/Pstream/mpi/UPstreamGatherScatter.C b/src/Pstream/mpi/UPstreamGatherScatter.C index 0a39fd71d6..561a1d41c9 100644 --- a/src/Pstream/mpi/UPstreamGatherScatter.C +++ b/src/Pstream/mpi/UPstreamGatherScatter.C @@ -257,59 +257,4 @@ void Foam::UPstream::mpi_scatterv } -// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // - -#undef Pstream_CommonRoutines -#define Pstream_CommonRoutines(Native, TaggedType) \ - \ -void Foam::UPstream::mpiGatherv \ -( \ - const Native* sendData, \ - int sendCount, \ - \ - Native* recvData, \ - const UList& recvCounts, \ - const UList& recvOffsets, \ - const label comm \ -) \ -{ \ - PstreamDetail::gatherv \ - ( \ - sendData, sendCount, \ - recvData, recvCounts, recvOffsets, \ - TaggedType, comm \ - ); \ -} \ - \ -void Foam::UPstream::mpiScatterv \ -( \ - const Native* sendData, \ - const UList& sendCounts, \ - const UList& sendOffsets, \ - \ - Native* recvData, \ - int recvCount, \ - const label comm \ -) \ -{ \ - PstreamDetail::scatterv \ - ( \ - sendData, sendCounts, sendOffsets, \ - recvData, recvCount, \ - TaggedType, comm \ - ); \ -} - - -//TDB: Pstream_CommonRoutines(bool, MPI_C_BOOL); -Pstream_CommonRoutines(char, MPI_BYTE); -Pstream_CommonRoutines(int32_t, MPI_INT32_T); -Pstream_CommonRoutines(int64_t, MPI_INT64_T); -Pstream_CommonRoutines(uint32_t, MPI_UINT32_T); -Pstream_CommonRoutines(uint64_t, MPI_UINT64_T); -Pstream_CommonRoutines(float, MPI_FLOAT); -Pstream_CommonRoutines(double, MPI_DOUBLE); - -#undef Pstream_CommonRoutines - // ************************************************************************* //