mirror of
https://develop.openfoam.com/Development/openfoam.git
synced 2025-11-28 03:28:01 +00:00
1ce7a62209
- dynamic sparse data exchange using Map to hold data and sizes. Still uses the personalised exchange paradigm, but with non-blocking consensus exchange to obtain the sizes and regular point-to-point for the data exchange itself. This avoids an all-to-all but still keeps the point-to-point for overlapping communication, data chunking etc.
641 lines
21 KiB
C++
641 lines
21 KiB
C++
/*---------------------------------------------------------------------------*\
|
|
========= |
|
|
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
|
|
\\ / O peration |
|
|
\\ / A nd | www.openfoam.com
|
|
\\/ M anipulation |
|
|
-------------------------------------------------------------------------------
|
|
Copyright (C) 2011-2016 OpenFOAM Foundation
|
|
Copyright (C) 2016-2023 OpenCFD Ltd.
|
|
-------------------------------------------------------------------------------
|
|
License
|
|
This file is part of OpenFOAM.
|
|
|
|
OpenFOAM is free software: you can redistribute it and/or modify it
|
|
under the terms of the GNU General Public License as published by
|
|
the Free Software Foundation, either version 3 of the License, or
|
|
(at your option) any later version.
|
|
|
|
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
|
|
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
|
for more details.
|
|
|
|
You should have received a copy of the GNU General Public License
|
|
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
Class
|
|
Foam::Pstream
|
|
|
|
Description
|
|
Inter-processor communications stream.
|
|
|
|
SourceFiles
|
|
Pstream.C
|
|
PstreamBroadcast.C
|
|
PstreamGather.C
|
|
PstreamCombineGather.C
|
|
PstreamGatherList.C
|
|
PstreamExchangeConsensus.C
|
|
PstreamExchange.C
|
|
|
|
\*---------------------------------------------------------------------------*/
|
|
|
|
#ifndef Foam_Pstream_H
|
|
#define Foam_Pstream_H
|
|
|
|
#include "UPstream.H"
|
|
#include "DynamicList.H"
|
|
|
|
// Legacy
|
|
// #define Foam_Pstream_scatter_nobroadcast
|
|
|
|
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
|
|
|
|
namespace Foam
|
|
{
|
|
|
|
/*---------------------------------------------------------------------------*\
|
|
Class Pstream Declaration
|
|
\*---------------------------------------------------------------------------*/
|
|
|
|
class Pstream
|
|
:
|
|
public UPstream
|
|
{
|
|
protected:
|
|
|
|
// Protected Data
|
|
|
|
//- Allocated transfer buffer (can be used for send or receive)
|
|
DynamicList<char> transferBuf_;
|
|
|
|
|
|
public:
|
|
|
|
//- Declare name of the class and its debug switch
|
|
ClassName("Pstream");
|
|
|
|
|
|
// Constructors
|
|
|
|
//- Construct for given communication type, with optional buffer size
|
|
explicit Pstream
|
|
(
|
|
const UPstream::commsTypes commsType,
|
|
const label bufSize = 0
|
|
)
|
|
:
|
|
UPstream(commsType)
|
|
{
|
|
if (bufSize > 0)
|
|
{
|
|
transferBuf_.setCapacity(bufSize + 2*sizeof(scalar) + 1);
|
|
}
|
|
}
|
|
|
|
|
|
// Static Functions
|
|
|
|
// Broadcast
|
|
|
|
//- Broadcast buffer content to all processes in communicator.
|
|
using UPstream::broadcast;
|
|
|
|
//- Broadcast content (contiguous or non-contiguous)
|
|
//- to all processes in communicator.
|
|
// For \b non-parallel : do nothing.
|
|
template<class Type>
|
|
static void broadcast
|
|
(
|
|
Type& value,
|
|
const label comm = UPstream::worldComm
|
|
);
|
|
|
|
//- Broadcast multiple items to all processes in communicator.
|
|
// For \b non-parallel : do nothing.
|
|
template<class Type, class... Args>
|
|
static void broadcasts(const label comm, Type& arg1, Args&&... args);
|
|
|
|
|
|
// Gather
|
|
|
|
//- Gather (reduce) data, appyling \c bop to combine \c value
|
|
//- from different processors. The basis for Foam::reduce().
|
|
// Uses the specified communication schedule.
|
|
template<class T, class BinaryOp>
|
|
static void gather
|
|
(
|
|
const List<commsStruct>& comms,
|
|
T& value,
|
|
const BinaryOp& bop,
|
|
const int tag,
|
|
const label comm
|
|
);
|
|
|
|
//- Gather (reduce) data, applying \c bop to combine \c value
|
|
//- from different processors. The basis for Foam::reduce().
|
|
// Uses linear/tree communication.
|
|
template<class T, class BinaryOp>
|
|
static void gather
|
|
(
|
|
T& value,
|
|
const BinaryOp& bop,
|
|
const int tag = UPstream::msgType(),
|
|
const label comm = UPstream::worldComm
|
|
);
|
|
|
|
|
|
// Gather/combine data
|
|
// Inplace combine values from processors.
|
|
// (Uses construct from Istream instead of <<)
|
|
|
|
//- Gather data, applying \c cop to inplace combine \c value
|
|
//- from different processors.
|
|
// Uses the specified communication schedule.
|
|
template<class T, class CombineOp>
|
|
static void combineGather
|
|
(
|
|
const List<commsStruct>& comms,
|
|
T& value,
|
|
const CombineOp& cop,
|
|
const int tag,
|
|
const label comm
|
|
);
|
|
|
|
//- Gather data, applying \c cop to inplace combine \c value
|
|
//- from different processors.
|
|
// Uses linear/tree communication.
|
|
template<class T, class CombineOp>
|
|
static void combineGather
|
|
(
|
|
T& value,
|
|
const CombineOp& cop,
|
|
const int tag = UPstream::msgType(),
|
|
const label comm = UPstream::worldComm
|
|
);
|
|
|
|
//- Reduce inplace (cf. MPI Allreduce)
|
|
//- applying \c cop to inplace combine \c value
|
|
//- from different processors.
|
|
//- After completion all processors have the same data.
|
|
// Uses the specified communication schedule.
|
|
// Wraps combineGather/broadcast (may change in the future).
|
|
template<class T, class CombineOp>
|
|
static void combineReduce
|
|
(
|
|
const List<commsStruct>& comms,
|
|
T& value,
|
|
const CombineOp& cop,
|
|
const int tag = UPstream::msgType(),
|
|
const label comm = UPstream::worldComm
|
|
);
|
|
|
|
//- Reduce inplace (cf. MPI Allreduce)
|
|
//- applying \c cop to inplace combine \c value
|
|
//- from different processors.
|
|
//- After completion all processors have the same data.
|
|
// Uses linear/tree communication.
|
|
// Wraps combineGather/broadcast (may change in the future).
|
|
template<class T, class CombineOp>
|
|
static void combineReduce
|
|
(
|
|
T& value,
|
|
const CombineOp& cop,
|
|
const int tag = UPstream::msgType(),
|
|
const label comm = UPstream::worldComm
|
|
);
|
|
|
|
//- Same as Pstream::combineReduce
|
|
template<class T, class CombineOp>
|
|
static void combineAllGather
|
|
(
|
|
T& value,
|
|
const CombineOp& cop,
|
|
const int tag = UPstream::msgType(),
|
|
const label comm = UPstream::worldComm
|
|
)
|
|
{
|
|
Pstream::combineReduce(value, cop, tag, comm);
|
|
}
|
|
|
|
|
|
// Combine variants working on whole List at a time.
|
|
|
|
template<class T, class CombineOp>
|
|
static void listCombineGather
|
|
(
|
|
const List<commsStruct>& comms,
|
|
List<T>& values,
|
|
const CombineOp& cop,
|
|
const int tag,
|
|
const label comm
|
|
);
|
|
|
|
//- Like above but switches between linear/tree communication
|
|
template<class T, class CombineOp>
|
|
static void listCombineGather
|
|
(
|
|
List<T>& values,
|
|
const CombineOp& cop,
|
|
const int tag = UPstream::msgType(),
|
|
const label comm = UPstream::worldComm
|
|
);
|
|
|
|
//- After completion all processors have the same data.
|
|
template<class T, class CombineOp>
|
|
static void listCombineReduce
|
|
(
|
|
List<T>& values,
|
|
const CombineOp& cop,
|
|
const int tag = UPstream::msgType(),
|
|
const label comm = UPstream::worldComm
|
|
);
|
|
|
|
//- Same as Pstream::listCombineReduce
|
|
template<class T, class CombineOp>
|
|
static void listCombineAllGather
|
|
(
|
|
List<T>& values,
|
|
const CombineOp& cop,
|
|
const int tag = UPstream::msgType(),
|
|
const label comm = UPstream::worldComm
|
|
)
|
|
{
|
|
Pstream::listCombineReduce(values, cop, tag, comm);
|
|
}
|
|
|
|
|
|
// Combine variants working on whole map at a time.
|
|
// Container needs iterators, find() and insert methods defined.
|
|
|
|
template<class Container, class CombineOp>
|
|
static void mapCombineGather
|
|
(
|
|
const List<commsStruct>& comms,
|
|
Container& values,
|
|
const CombineOp& cop,
|
|
const int tag,
|
|
const label comm
|
|
);
|
|
|
|
//- Like above but switches between linear/tree communication
|
|
template<class Container, class CombineOp>
|
|
static void mapCombineGather
|
|
(
|
|
Container& values,
|
|
const CombineOp& cop,
|
|
const int tag = UPstream::msgType(),
|
|
const label comm = UPstream::worldComm
|
|
);
|
|
|
|
//- Reduce inplace (cf. MPI Allreduce)
|
|
//- applying \c cop to inplace combine map \c values
|
|
//- from different processors.
|
|
//- After completion all processors have the same data.
|
|
// Uses the specified communication schedule.
|
|
// Wraps mapCombineGather/broadcast (may change in the future).
|
|
//- After completion all processors have the same data.
|
|
template<class Container, class CombineOp>
|
|
static void mapCombineReduce
|
|
(
|
|
Container& values,
|
|
const CombineOp& cop,
|
|
const int tag = UPstream::msgType(),
|
|
const label comm = UPstream::worldComm
|
|
);
|
|
|
|
//- Same as Pstream::mapCombineReduce
|
|
template<class Container, class CombineOp>
|
|
static void mapCombineAllGather
|
|
(
|
|
Container& values,
|
|
const CombineOp& cop,
|
|
const int tag = UPstream::msgType(),
|
|
const label comm = UPstream::worldComm
|
|
)
|
|
{
|
|
Pstream::mapCombineReduce(values, cop, tag, comm);
|
|
}
|
|
|
|
|
|
// Gather/scatter keeping the individual processor data separate.
|
|
// The values is a List of size UPstream::nProcs() where
|
|
// values[UPstream::myProcNo()] is the data for the current processor.
|
|
|
|
//- Gather data, but keep individual values separate.
|
|
//- Uses the specified communication schedule.
|
|
template<class T>
|
|
static void gatherList
|
|
(
|
|
const List<commsStruct>& comms,
|
|
List<T>& values,
|
|
const int tag,
|
|
const label comm
|
|
);
|
|
|
|
//- Gather data, but keep individual values separate.
|
|
//- Uses linear/tree communication.
|
|
template<class T>
|
|
static void gatherList
|
|
(
|
|
List<T>& values,
|
|
const int tag = UPstream::msgType(),
|
|
const label comm = UPstream::worldComm
|
|
);
|
|
|
|
//- Gather data, but keep individual values separate.
|
|
//- Uses linear/tree communication.
|
|
// After completion all processors have the same data.
|
|
// Wraps gatherList/scatterList (may change in the future).
|
|
template<class T>
|
|
static void allGatherList
|
|
(
|
|
List<T>& values,
|
|
const int tag = UPstream::msgType(),
|
|
const label comm = UPstream::worldComm
|
|
);
|
|
|
|
|
|
// Scatter
|
|
|
|
//- Broadcast data: Distribute without modification.
|
|
// \note comms and tag parameters only used when
|
|
// Foam_Pstream_scatter_nobroadcast is defined
|
|
template<class T>
|
|
static void scatter
|
|
(
|
|
const List<commsStruct>& comms,
|
|
T& value,
|
|
const int tag,
|
|
const label comm
|
|
);
|
|
|
|
//- Broadcast data: Distribute without modification.
|
|
// \note tag parameter only used when
|
|
// Foam_Pstream_scatter_nobroadcast is defined
|
|
template<class T>
|
|
static void scatter
|
|
(
|
|
T& value,
|
|
const int tag = UPstream::msgType(),
|
|
const label comm = UPstream::worldComm
|
|
);
|
|
|
|
//- Broadcast data: Distribute without modification.
|
|
// \note tag parameter only used when
|
|
// Foam_Pstream_scatter_nobroadcast is defined
|
|
template<class T>
|
|
static void combineScatter
|
|
(
|
|
const List<commsStruct>& comms,
|
|
T& value,
|
|
const int tag,
|
|
const label comm
|
|
);
|
|
|
|
//- Broadcast data: Distribute without modification.
|
|
// \note tag parameter only used when
|
|
// Foam_Pstream_scatter_nobroadcast is defined
|
|
template<class T>
|
|
static void combineScatter
|
|
(
|
|
T& value,
|
|
const int tag = UPstream::msgType(),
|
|
const label comm = UPstream::worldComm
|
|
);
|
|
|
|
//- Broadcast data: Distribute without modification.
|
|
// \note comms and tag parameters only used when
|
|
// Foam_Pstream_scatter_nobroadcast is defined
|
|
template<class T>
|
|
static void listCombineScatter
|
|
(
|
|
const List<commsStruct>& comms,
|
|
List<T>& value,
|
|
const int tag,
|
|
const label comm
|
|
);
|
|
|
|
//- Broadcast data: Distribute without modification.
|
|
// \note comms and tag parameters only used when
|
|
// Foam_Pstream_scatter_nobroadcast is defined
|
|
template<class T>
|
|
static void listCombineScatter
|
|
(
|
|
List<T>& value,
|
|
const int tag = UPstream::msgType(),
|
|
const label comm = UPstream::worldComm
|
|
);
|
|
|
|
//- Broadcast data: Distribute without modification.
|
|
template<class Container>
|
|
static void mapCombineScatter
|
|
(
|
|
const List<commsStruct>& comms,
|
|
Container& values,
|
|
const int tag,
|
|
const label comm
|
|
);
|
|
|
|
//- Like above but switches between linear/tree communication
|
|
template<class Container>
|
|
static void mapCombineScatter
|
|
(
|
|
Container& values,
|
|
const int tag = UPstream::msgType(),
|
|
const label comm = UPstream::worldComm
|
|
);
|
|
|
|
|
|
//- Scatter data. Reverse of gatherList
|
|
template<class T>
|
|
static void scatterList
|
|
(
|
|
const List<commsStruct>& comms,
|
|
List<T>& values,
|
|
const int tag,
|
|
const label comm
|
|
);
|
|
|
|
//- Like above but switches between linear/tree communication
|
|
template<class T>
|
|
static void scatterList
|
|
(
|
|
List<T>& values,
|
|
const int tag = UPstream::msgType(),
|
|
const label comm = UPstream::worldComm
|
|
);
|
|
|
|
|
|
// Exchange
|
|
|
|
//- Helper: exchange sizes of sendBufs for specified
|
|
//- set of send/receive processes.
|
|
template<class Container>
|
|
static void exchangeSizes
|
|
(
|
|
const labelUList& sendProcs,
|
|
const labelUList& recvProcs,
|
|
const Container& sendBufs,
|
|
labelList& sizes,
|
|
const label tag = UPstream::msgType(),
|
|
const label comm = UPstream::worldComm
|
|
);
|
|
|
|
//- Helper: exchange sizes of sendBufs.
|
|
//- The sendBufs is the data per processor (in the communicator).
|
|
// Returns sizes of sendBufs on the sending processor.
|
|
// \n
|
|
// For \b non-parallel : copy sizes from sendBufs directly.
|
|
template<class Container>
|
|
static void exchangeSizes
|
|
(
|
|
const Container& sendBufs,
|
|
labelList& recvSizes,
|
|
const label comm = UPstream::worldComm
|
|
);
|
|
|
|
//- Exchange the \b non-zero sizes of sendBufs entries (sparse map)
|
|
//- with other ranks in the communicator
|
|
//- using non-blocking consensus exchange.
|
|
//
|
|
// Since the recvData map always cleared before receipt and sizes
|
|
// of zero are never transmitted, a simple check
|
|
// of its keys is sufficient to determine connectivity.
|
|
//
|
|
// For \b non-parallel : copy size of rank (if it exists and non-empty)
|
|
// from sendBufs to recvSizes.
|
|
//
|
|
// \note The message tag is adjusted internally to improve uniqueness
|
|
template<class Container>
|
|
static void exchangeSizes
|
|
(
|
|
const Map<Container>& sendBufs,
|
|
Map<label>& recvSizes,
|
|
const label tag = UPstream::msgType(),
|
|
const label comm = UPstream::worldComm
|
|
);
|
|
|
|
//- Helper: exchange \em contiguous data.
|
|
//- Sends sendBufs, receives into recvBufs using predetermined receive
|
|
//- sizing.
|
|
// If wait=true will wait for all transfers to finish.
|
|
template<class Container, class Type>
|
|
static void exchange
|
|
(
|
|
const UList<Container>& sendBufs,
|
|
const labelUList& recvSizes,
|
|
List<Container>& recvBufs,
|
|
const int tag = UPstream::msgType(),
|
|
const label comm = UPstream::worldComm,
|
|
const bool wait = true //!< Wait for requests to complete
|
|
);
|
|
|
|
//- Exchange \em contiguous data.
|
|
//- Sends sendBufs, receives into recvBufs.
|
|
// Data provided and received as container.
|
|
//
|
|
// No internal guards or resizing.
|
|
template<class Container, class Type>
|
|
static void exchange
|
|
(
|
|
const Map<Container>& sendBufs,
|
|
const Map<label>& recvSizes, //!< Num of recv elements (not bytes)
|
|
Map<Container>& recvBufs,
|
|
const int tag = UPstream::msgType(),
|
|
const label comm = UPstream::worldComm,
|
|
const bool wait = true //!< Wait for requests to complete
|
|
);
|
|
|
|
//- Exchange \em contiguous data.
|
|
//- Sends sendBufs, receives into recvBufs.
|
|
//- Determines sizes to receive.
|
|
// If wait=true will wait for all transfers to finish.
|
|
template<class Container, class Type>
|
|
static void exchange
|
|
(
|
|
const UList<Container>& sendBufs,
|
|
List<Container>& recvBufs,
|
|
const int tag = UPstream::msgType(),
|
|
const label comm = UPstream::worldComm,
|
|
const bool wait = true //!< Wait for requests to complete
|
|
);
|
|
|
|
//- Exchange \em contiguous data.
|
|
//- Sends sendBufs, receives into recvBufs.
|
|
//- Determines sizes to receive.
|
|
// If wait=true will wait for all transfers to finish.
|
|
template<class Container, class Type>
|
|
static void exchange
|
|
(
|
|
const Map<Container>& sendBufs,
|
|
Map<Container>& recvBufs,
|
|
const int tag = UPstream::msgType(),
|
|
const label comm = UPstream::worldComm,
|
|
const bool wait = true //!< Wait for requests to complete
|
|
);
|
|
|
|
|
|
// Non-blocking exchange
|
|
|
|
//- Exchange \em contiguous data using non-blocking consensus
|
|
//- Sends sendData, receives into recvData.
|
|
//
|
|
// Each entry of the recvBufs list is cleared before receipt.
|
|
// For \b non-parallel : copy own rank from sendBufs to recvBufs.
|
|
//
|
|
// \note The message tag should be chosen to be a unique value
|
|
// since the implementation uses probing with ANY_SOURCE !!
|
|
template<class Container, class Type>
|
|
static void exchangeConsensus
|
|
(
|
|
const UList<Container>& sendBufs,
|
|
List<Container>& recvBufs,
|
|
const int tag,
|
|
const label comm
|
|
);
|
|
|
|
//- Exchange \em contiguous data using non-blocking consensus
|
|
//- Sends sendData, receives into recvData.
|
|
//
|
|
// Each \em entry of the recvBufs map is cleared before receipt,
|
|
// but the map itself if not cleared. This allows the map to preserve
|
|
// allocated space (eg DynamicList entries) between calls.
|
|
//
|
|
// For \b non-parallel : copy own rank (if it exists and non-empty)
|
|
// from sendBufs to recvBufs.
|
|
//
|
|
// \note The message tag should be chosen to be a unique value
|
|
// since the implementation uses probing with ANY_SOURCE !!
|
|
template<class Container, class Type>
|
|
static void exchangeConsensus
|
|
(
|
|
const Map<Container>& sendBufs,
|
|
Map<Container>& recvBufs,
|
|
const int tag,
|
|
const label comm
|
|
);
|
|
};
|
|
|
|
|
|
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
|
|
|
|
} // End namespace Foam
|
|
|
|
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
|
|
|
|
#ifdef NoRepository
|
|
#include "PstreamBroadcast.C"
|
|
#include "PstreamGather.C"
|
|
#include "PstreamCombineGather.C"
|
|
#include "PstreamGatherList.C"
|
|
#include "PstreamExchange.C"
|
|
#endif
|
|
|
|
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
|
|
|
|
#endif
|
|
|
|
// ************************************************************************* //
|