zhyang-dev/openfoam
1
0
Fork 0
mirror of https://develop.openfoam.com/Development/openfoam.git synced 2025-11-28 03:28:01 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

ENH: Pstream specialization for float/scalar, FixedList (#2351)

Browse Source 
- native MPI min/max/sum reductions for float/double
  irrespective of WM_PRECISION_OPTION

- native MPI min/max/sum reductions for (u)int32_t/(u)int64_t types,
  irrespective of WM_LABEL_SIZE

- replace rarely used vector2D sum reduction with FixedList as a
  indicator of its intent and also generalizes to different lengths.

  OLD:
      vector2D values;  values.x() = ...;  values.y() = ...;
      reduce(values, sumOp<vector2D>());

  NEW:
      FixedList<scalar,2> values;  values[0] = ...;  values[1] = ...;
      reduce(values, sumOp<scalar>());

- allow returnReduce() to use native reductions. Previous code (with
  linear/tree selector) would have bypassed them inadvertently.

ENH: added support for MPI broadcast (for a memory span)

ENH: select communication schedule as a static method

- UPstream::whichCommunication(comm) to select linear/tree
  communication instead of ternary or
  if (Pstream::nProcs() < Pstream::nProcsSimpleSum) ...

STYLE: align nProcsSimpleSum static value with etc/controlDict override
This commit is contained in:
Mark Olesen
2022-02-14 12:12:32 +01:00
committed by Andrew Heather
parent b95b24e4e7
commit b0ef650a12
24 changed files with 932 additions and 940 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
applications/test/parallel-communicators/Test-parallel-communicators.C
View File

@ -163,16 +163,6 @@ int main(int argc, char *argv[])
if (Pstream::myProcNo(comm) != -1) if (Pstream::myProcNo(comm) != -1)
{ {
//scalar sum = sumReduce(comm, localValue);
//scalar sum = localValue;
//reduce
//(
// UPstream::treeCommunication(comm),
// sum,
// sumOp<scalar>(),
// Pstream::msgType(),
// comm
//);
scalar sum = returnReduce scalar sum = returnReduce
( (
localValue, localValue,

45
src/OpenFOAM/db/IOstreams/Pstreams/PstreamCombineReduceOps.H
View File

@ -35,8 +35,8 @@ Description
\*---------------------------------------------------------------------------*/ \*---------------------------------------------------------------------------*/
#ifndef PstreamCombineReduceOps_H #ifndef Foam_PstreamCombineReduceOps_H
#define PstreamCombineReduceOps_H #define Foam_PstreamCombineReduceOps_H
#include "UPstream.H" #include "UPstream.H"
#include "Pstream.H" #include "Pstream.H"
@ -73,42 +73,11 @@ void combineReduce
const label comm = Pstream::worldComm const label comm = Pstream::worldComm
) )
{ {
if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum) const List<UPstream::commsStruct>& comms =
{ UPstream::whichCommunication(comm);
Pstream::combineGather
( Pstream::combineGather(comms, Value, cop, tag, comm);
UPstream::linearCommunication(comm), Pstream::combineScatter(comms, Value, tag, comm);
Value,
cop,
tag,
comm
);
Pstream::combineScatter
(
UPstream::linearCommunication(comm),
Value,
tag,
comm
);
}
else
{
Pstream::combineGather
(
UPstream::treeCommunication(comm),
Value,
cop,
tag,
comm
);
Pstream::combineScatter
(
UPstream::treeCommunication(comm),
Value,
tag,
comm
);
}
} }

311
src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H
View File

@ -6,7 +6,7 @@
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2011-2016 OpenFOAM Foundation Copyright (C) 2011-2016 OpenFOAM Foundation
Copyright (C) 2016 OpenCFD Ltd. Copyright (C) 2016-2022 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
@ -32,11 +32,11 @@ Description
\*---------------------------------------------------------------------------*/ \*---------------------------------------------------------------------------*/
#ifndef PstreamReduceOps_H #ifndef Foam_PstreamReduceOps_H
#define PstreamReduceOps_H #define Foam_PstreamReduceOps_H
#include "ops.H" #include "ops.H"
#include "vector2D.H" #include "FixedList.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
@ -45,12 +45,12 @@ namespace Foam
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
// Reduce operation with user specified communication schedule //- Reduce operation with user specified communication schedule
template<class T, class BinaryOp> template<class T, class BinaryOp>
void reduce void reduce
( (
const List<UPstream::commsStruct>& comms, const List<UPstream::commsStruct>& comms,
T& Value, T& value,
const BinaryOp& bop, const BinaryOp& bop,
const int tag, const int tag,
const label comm const label comm
@ -58,96 +58,86 @@ void reduce
{ {
if (UPstream::warnComm != -1 && comm != UPstream::warnComm) if (UPstream::warnComm != -1 && comm != UPstream::warnComm)
{ {
Pout<< "** reducing:" << Value << " with comm:" << comm Pout<< "** reducing:" << value << " with comm:" << comm
<< endl; << endl;
error::printStack(Pout); error::printStack(Pout);
} }
Pstream::gather(comms, Value, bop, tag, comm); Pstream::gather(comms, value, bop, tag, comm);
Pstream::scatter(comms, Value, tag, comm); Pstream::scatter(comms, value, tag, comm);
} }
// Reduce using either linear or tree communication schedule //- Reduce (inplace) using either linear or tree communication schedule
template<class T, class BinaryOp> template<class T, class BinaryOp>
void reduce void reduce
( (
T& Value, T& value,
const BinaryOp& bop, const BinaryOp& bop,
const int tag = Pstream::msgType(), const int tag = UPstream::msgType(),
const label comm = UPstream::worldComm const label comm = UPstream::worldComm
) )
{ {
if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum) if (UPstream::parRun())
{ {
reduce(UPstream::linearCommunication(comm), Value, bop, tag, comm); reduce(UPstream::whichCommunication(comm), value, bop, tag, comm);
}
else
{
reduce(UPstream::treeCommunication(comm), Value, bop, tag, comm);
} }
} }
// Reduce using either linear or tree communication schedule //- Reduce (copy) and return value
template<class T, class BinaryOp> template<class T, class BinaryOp>
T returnReduce T returnReduce
( (
const T& Value, const T& value,
const BinaryOp& bop, const BinaryOp& bop,
const int tag = Pstream::msgType(), const int tag = UPstream::msgType(),
const label comm = UPstream::worldComm const label comm = UPstream::worldComm
) )
{ {
T WorkValue(Value); T work(value);
reduce(work, bop, tag, comm);
if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum) return work;
{
reduce
(
UPstream::linearCommunication(comm),
WorkValue,
bop,
tag,
comm
);
}
else
{
reduce
(
UPstream::treeCommunication(comm),
WorkValue,
bop,
tag,
comm
);
}
return WorkValue;
} }
// Reduce with sum of both value and count (for averaging) //- Reduce with sum of both value and count (for averaging)
template<class T> template<class T>
void sumReduce void sumReduce
( (
T& Value, T& value,
label& Count, label& count,
const int tag = Pstream::msgType(), const int tag = UPstream::msgType(),
const label comm = UPstream::worldComm const label comm = UPstream::worldComm
) )
{ {
reduce(Value, sumOp<T>(), tag, comm); if (UPstream::parRun())
reduce(Count, sumOp<label>(), tag, comm); {
reduce(value, sumOp<T>(), tag, comm);
reduce(count, sumOp<label>(), tag, comm);
}
} }
// Non-blocking version of reduce. Sets request. //- Reduce multiple values (identical size on all processes!)
template<class T, class BinaryOp>
void reduce
(
T values[],
const int size,
const BinaryOp&,
const int tag,
const label comm
)
{
NotImplemented;
}
//- Non-blocking reduce single value. Sets request.
template<class T, class BinaryOp> template<class T, class BinaryOp>
void reduce void reduce
( (
T& Value, T& Value,
const BinaryOp& bop, const BinaryOp&,
const int tag, const int tag,
const label comm, const label comm,
label& request label& request
@ -156,13 +146,14 @@ void reduce
NotImplemented; NotImplemented;
} }
// Non-blocking version of reduce. Sets request. //- Non-blocking reduce multiple values (identical size on all processes!)
//- Sets request.
template<class T, class BinaryOp> template<class T, class BinaryOp>
void reduce void reduce
( (
T Value[], T values[],
const int size, const int size,
const BinaryOp& bop, const BinaryOp&,
const int tag, const int tag,
const label comm, const label comm,
label& request label& request
@ -172,111 +163,139 @@ void reduce
} }
// Insist there are specialisations for the common reductions of scalar(s) // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
// Specialisations for bool
//- Logical (and) reduction
void reduce void reduce
( (
scalar& Value, bool& value,
const sumOp<scalar>& bop, const andOp<bool>&,
const int tag = Pstream::msgType(), const int tag = UPstream::msgType(),
const label comm = UPstream::worldComm const label comm = UPstream::worldComm
); );
//- Logical (or) reduction
void reduce void reduce
( (
scalar& Value, bool& value,
const minOp<scalar>& bop, const orOp<bool>&,
const int tag = Pstream::msgType(), const int tag = UPstream::msgType(),
const label comm = UPstream::worldComm const label comm = UPstream::worldComm
); );
void reduce
(
vector2D& Value,
const sumOp<vector2D>& bop,
const int tag = Pstream::msgType(),
const label comm = UPstream::worldComm
);
void sumReduce // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
(
scalar& Value,
label& Count,
const int tag = Pstream::msgType(),
const label comm = UPstream::worldComm
);
void reduce // Specialisations for common reduction types
(
scalar& Value,
const sumOp<scalar>& bop,
const int tag,
const label comm,
label& request
);
void reduce #undef Pstream_CommonReductions
( #define Pstream_CommonReductions(Native) \
scalar Value[], \
const int size, /*! \brief Reduce (min) single Native value */ \
const sumOp<scalar>& bop, void reduce \
const int tag, ( \
const label comm, Native& value, \
label& request const minOp<Native>&, \
const int tag = UPstream::msgType(), \
const label comm = UPstream::worldComm \
); \
\
/*! \brief Reduce (max) single Native value */ \
void reduce \
( \
Native& value, \
const maxOp<Native>&, \
const int tag = UPstream::msgType(), \
const label comm = UPstream::worldComm \
); \
\
/*! \brief Reduce (sum) single Native value */ \
void reduce \
( \
Native& value, \
const sumOp<Native>&, \
const int tag = UPstream::msgType(), \
const label comm = UPstream::worldComm \
); \
\
/*! \brief Reduce (sum) multiple Native values (identical size all procs!) */ \
void reduce \
( \
Native values[], \
const int size, \
const sumOp<Native>&, \
const int tag, \
const label comm \
); \
\
/*! \brief Reduce (sum) multiple Native values */ \
template<unsigned N> \
inline void reduce \
( \
FixedList<Native, N>& values, \
const sumOp<Native>&, \
const int tag = UPstream::msgType(), \
const label comm = UPstream::worldComm \
) \
{ \
reduce(values.data(), int(values.size()), sumOp<Native>(), tag, comm); \
}
Pstream_CommonReductions(int32_t);
Pstream_CommonReductions(int64_t);
Pstream_CommonReductions(uint32_t);
Pstream_CommonReductions(uint64_t);
Pstream_CommonReductions(float);
Pstream_CommonReductions(double);
#undef Pstream_CommonReductions
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
// Specialisations for floating-point types
#undef Pstream_FloatReductions
#define Pstream_FloatReductions(Native) \
\
/*! \brief Sum of both Native value and count (for averaging) */ \
void sumReduce \
( \
Native& value, \
label& count, \
const int tag = UPstream::msgType(), \
const label comm = UPstream::worldComm \
); \
\
/*! \brief Non-blocking reduce (sum) single Native value. Sets request */ \
void reduce \
( \
Native& value, \
const sumOp<Native>&, \
const int tag, \
const label comm, \
label& requestID \
); \
\
/*! \brief Non-blocking reduce (sum) multiple Native values. Sets request */ \
void reduce \
( \
Native values[], \
const int size, \
const sumOp<Native>&, \
const int tag, \
const label comm, \
label& requestID \
); );
#if defined(WM_SPDP) Pstream_FloatReductions(float);
void reduce Pstream_FloatReductions(double);
(
solveScalar& Value,
const sumOp<solveScalar>& bop,
const int tag = Pstream::msgType(),
const label comm = UPstream::worldComm
);
void reduce #undef Pstream_FloatReductions
(
solveScalar& Value,
const minOp<solveScalar>& bop,
const int tag = Pstream::msgType(),
const label comm = UPstream::worldComm
);
void reduce
(
Vector2D<solveScalar>& Value,
const sumOp<Vector2D<solveScalar>>& bop,
const int tag = Pstream::msgType(),
const label comm = UPstream::worldComm
);
void sumReduce
(
solveScalar& Value,
label& Count,
const int tag = Pstream::msgType(),
const label comm = UPstream::worldComm
);
void reduce
(
solveScalar& Value,
const sumOp<solveScalar>& bop,
const int tag,
const label comm,
label& request
);
void reduce
(
solveScalar Value[],
const int size,
const sumOp<solveScalar>& bop,
const int tag,
const label comm,
label& request
);
#endif
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //

4
src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2015-2021 OpenCFD Ltd. Copyright (C) 2015-2022 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
@ -414,7 +414,7 @@ registerOptSwitch
int Foam::UPstream::nProcsSimpleSum int Foam::UPstream::nProcsSimpleSum
( (
Foam::debug::optimisationSwitch("nProcsSimpleSum", 16) Foam::debug::optimisationSwitch("nProcsSimpleSum", 0)
); );
registerOptSwitch registerOptSwitch
( (

42
src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H
View File

@ -69,9 +69,9 @@ public:
//- Types of communications //- Types of communications
enum class commsTypes : char enum class commsTypes : char
{ {
blocking, blocking, //!< "blocking"
scheduled, scheduled, //!< "scheduled"
nonBlocking nonBlocking //!< "nonBlocking"
}; };
//- Names of the communication types //- Names of the communication types
@ -413,11 +413,11 @@ public:
static int allocateTag(const char*); static int allocateTag(const char*);
static int allocateTag(const word&); static int allocateTag(const std::string&);
static void freeTag(const char*, const int tag); static void freeTag(const char*, const int tag);
static void freeTag(const word&, const int tag); static void freeTag(const std::string&, const int tag);
//- Set as parallel run on/off. //- Set as parallel run on/off.
@ -537,13 +537,28 @@ public:
return treeCommunication_[communicator]; return treeCommunication_[communicator];
} }
//- Communication schedule for linear/tree all-to-master (proc 0).
//- Chooses based on the value of UPstream::nProcsSimpleSum
static const List<commsStruct>& whichCommunication
(
const label communicator = worldComm
)
{
return
(
nProcs(communicator) < nProcsSimpleSum
? linearCommunication_[communicator]
: treeCommunication_[communicator]
);
}
//- Message tag of standard messages //- Message tag of standard messages
static int& msgType() noexcept static int& msgType() noexcept
{ {
return msgType_; return msgType_;
} }
//- Get the communications type of the stream //- Get the communications type of the stream
commsTypes commsType() const noexcept commsTypes commsType() const noexcept
{ {
@ -551,6 +566,7 @@ public:
} }
//- Set the communications type of the stream //- Set the communications type of the stream
// \return the previous value
commsTypes commsType(const commsTypes ct) noexcept commsTypes commsType(const commsTypes ct) noexcept
{ {
commsTypes old(commsType_); commsTypes old(commsType_);
@ -668,6 +684,20 @@ public:
); );
// Broadcast Functions
//- Broadcast buffer contents, sizes must match on processors
// \return True on success
static bool broadcast
(
char* buf,
const std::streamsize bufSize,
const label communicator = worldComm,
const int rootProcNo = masterNo()
);
// Housekeeping // Housekeeping
//- Process index of first sub-process //- Process index of first sub-process

89
src/OpenFOAM/db/IOstreams/Pstreams/combineGatherScatter.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2019-2021 OpenCFD Ltd. Copyright (C) 2019-2022 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
@ -152,30 +152,16 @@ void Foam::Pstream::combineGather
const int tag, const int tag,
const label comm const label comm
) )
{
if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
{ {
combineGather combineGather
( (
UPstream::linearCommunication(comm), UPstream::whichCommunication(comm),
Value, Value,
cop, cop,
tag, tag,
comm comm
); );
} }
else
{
combineGather
(
UPstream::treeCommunication(comm),
Value,
cop,
tag,
comm
);
}
}
template<class T> template<class T>
@ -274,14 +260,7 @@ void Foam::Pstream::combineScatter
const label comm const label comm
) )
{ {
if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum) combineScatter(UPstream::whichCommunication(comm), Value, tag, comm);
{
combineScatter(UPstream::linearCommunication(comm), Value, tag, comm);
}
else
{
combineScatter(UPstream::treeCommunication(comm), Value, tag, comm);
}
} }
@ -401,30 +380,16 @@ void Foam::Pstream::listCombineGather
const int tag, const int tag,
const label comm const label comm
) )
{
if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
{ {
listCombineGather listCombineGather
( (
UPstream::linearCommunication(comm), UPstream::whichCommunication(comm),
Values, Values,
cop, cop,
tag, tag,
comm comm
); );
} }
else
{
listCombineGather
(
UPstream::treeCommunication(comm),
Values,
cop,
tag,
comm
);
}
}
template<class T> template<class T>
@ -522,28 +487,15 @@ void Foam::Pstream::listCombineScatter
const int tag, const int tag,
const label comm const label comm
) )
{
if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
{ {
listCombineScatter listCombineScatter
( (
UPstream::linearCommunication(comm), UPstream::whichCommunication(comm),
Values, Values,
tag, tag,
comm comm
); );
} }
else
{
listCombineScatter
(
UPstream::treeCommunication(comm),
Values,
tag,
comm
);
}
}
template<class Container, class CombineOp> template<class Container, class CombineOp>
@ -635,30 +587,16 @@ void Foam::Pstream::mapCombineGather
const int tag, const int tag,
const label comm const label comm
) )
{
if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
{ {
mapCombineGather mapCombineGather
( (
UPstream::linearCommunication(comm), UPstream::whichCommunication(comm),
Values, Values,
cop, cop,
tag, tag,
comm comm
); );
} }
else
{
mapCombineGather
(
UPstream::treeCommunication(comm),
Values,
cop,
tag,
comm
);
}
}
template<class Container> template<class Container>
@ -726,28 +664,15 @@ void Foam::Pstream::mapCombineScatter
const int tag, const int tag,
const label comm const label comm
) )
{
if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
{ {
mapCombineScatter mapCombineScatter
( (
UPstream::linearCommunication(comm), UPstream::whichCommunication(comm),
Values, Values,
tag, tag,
comm comm
); );
} }
else
{
mapCombineScatter
(
UPstream::treeCommunication(comm),
Values,
tag,
comm
);
}
}
// ************************************************************************* // // ************************************************************************* //

20
src/OpenFOAM/db/IOstreams/Pstreams/gatherScatter.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2019 OpenCFD Ltd. Copyright (C) 2019-2022 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
@ -134,14 +134,7 @@ void Pstream::gather
const label comm const label comm
) )
{ {
if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum) gather(UPstream::whichCommunication(comm), Value, bop, tag, comm);
{
gather(UPstream::linearCommunication(comm), Value, bop, tag, comm);
}
else
{
gather(UPstream::treeCommunication(comm), Value, bop, tag, comm);
}
} }
@ -225,14 +218,7 @@ void Pstream::scatter
template<class T> template<class T>
void Pstream::scatter(T& Value, const int tag, const label comm) void Pstream::scatter(T& Value, const int tag, const label comm)
{ {
if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum) scatter(UPstream::whichCommunication(comm), Value, tag, comm);
{
scatter(UPstream::linearCommunication(comm), Value, tag, comm);
}
else
{
scatter(UPstream::treeCommunication(comm), Value, tag, comm);
}
} }

20
src/OpenFOAM/db/IOstreams/Pstreams/gatherScatterList.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2015-2021 OpenCFD Ltd. Copyright (C) 2015-2022 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
@ -199,14 +199,7 @@ void Pstream::gatherList
template<class T> template<class T>
void Pstream::gatherList(List<T>& Values, const int tag, const label comm) void Pstream::gatherList(List<T>& Values, const int tag, const label comm)
{ {
if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum) gatherList(UPstream::whichCommunication(comm), Values, tag, comm);
{
gatherList(UPstream::linearCommunication(comm), Values, tag, comm);
}
else
{
gatherList(UPstream::treeCommunication(comm), Values, tag, comm);
}
} }
@ -341,14 +334,7 @@ void Pstream::scatterList
template<class T> template<class T>
void Pstream::scatterList(List<T>& Values, const int tag, const label comm) void Pstream::scatterList(List<T>& Values, const int tag, const label comm)
{ {
if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum) scatterList(UPstream::whichCommunication(comm), Values, tag, comm);
{
scatterList(UPstream::linearCommunication(comm), Values, tag, comm);
}
else
{
scatterList(UPstream::treeCommunication(comm), Values, tag, comm);
}
} }

18
src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2017-2018 OpenFOAM Foundation Copyright (C) 2017-2018 OpenFOAM Foundation
Copyright (C) 2019-2021 OpenCFD Ltd. Copyright (C) 2019-2022 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
@ -2072,22 +2072,12 @@ bool Foam::fileOperations::masterUncollatedFileOperation::read
Pstream::scatter(io.note()); //, Pstream::msgType(), comm_); Pstream::scatter(io.note()); //, Pstream::msgType(), comm_);
// scatter operation for regIOobjects // Scatter operation for regIOobjects
// Get my communication order // Get my communication order
// const List<Pstream::commsStruct>& comms =
//(
// (Pstream::nProcs(comm_) < Pstream::nProcsSimpleSum)
// ? Pstream::linearCommunication(comm_)
// : Pstream::treeCommunication(comm_)
//);
// const Pstream::commsStruct& myComm = comms[Pstream::myProcNo(comm_)];
const List<Pstream::commsStruct>& comms = const List<Pstream::commsStruct>& comms =
( Pstream::whichCommunication(Pstream::worldComm);
(Pstream::nProcs(Pstream::worldComm) < Pstream::nProcsSimpleSum)
? Pstream::linearCommunication(Pstream::worldComm)
: Pstream::treeCommunication(Pstream::worldComm)
);
const Pstream::commsStruct& myComm = const Pstream::commsStruct& myComm =
comms[Pstream::myProcNo(Pstream::worldComm)]; comms[Pstream::myProcNo(Pstream::worldComm)];

9
src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2017 OpenFOAM Foundation Copyright (C) 2017 OpenFOAM Foundation
Copyright (C) 2020-2021 OpenCFD Ltd. Copyright (C) 2020-2022 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
@ -655,12 +655,7 @@ bool Foam::fileOperations::uncollatedFileOperation::read
Pstream::scatter(io.note()); Pstream::scatter(io.note());
// Get my communication order // Get my communication order
const List<Pstream::commsStruct>& comms = const List<Pstream::commsStruct>& comms = Pstream::whichCommunication();
(
(Pstream::nProcs() < Pstream::nProcsSimpleSum)
? Pstream::linearCommunication()
: Pstream::treeCommunication()
);
const Pstream::commsStruct& myComm = comms[Pstream::myProcNo()]; const Pstream::commsStruct& myComm = comms[Pstream::myProcNo()];
// Receive from up // Receive from up

20
src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverScale.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2017-2019 OpenCFD Ltd. Copyright (C) 2017-2022 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
@ -27,7 +27,7 @@ License
\*---------------------------------------------------------------------------*/ \*---------------------------------------------------------------------------*/
#include "GAMGSolver.H" #include "GAMGSolver.H"
#include "vector2D.H" #include "FixedList.H"
// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * // // * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
@ -58,21 +58,21 @@ void Foam::GAMGSolver::scale
const solveScalar* const __restrict__ AcfPtr = Acf.begin(); const solveScalar* const __restrict__ AcfPtr = Acf.begin();
solveScalar scalingFactorNum = 0.0; FixedList<solveScalar, 2> scalingFactor(Zero);
solveScalar scalingFactorDenom = 0.0;
for (label i=0; i<nCells; i++) for (label i=0; i<nCells; i++)
{ {
scalingFactorNum += sourcePtr[i]*fieldPtr[i]; scalingFactor[0] += fieldPtr[i]*sourcePtr[i];
scalingFactorDenom += AcfPtr[i]*fieldPtr[i]; scalingFactor[1] += fieldPtr[i]*AcfPtr[i];
} }
Vector2D<solveScalar> scalingVector(scalingFactorNum, scalingFactorDenom); A.mesh().reduce(scalingFactor, sumOp<solveScalar>());
A.mesh().reduce(scalingVector, sumOp<Vector2D<solveScalar>>());
const solveScalar sf = const solveScalar sf =
scalingVector.x() (
/stabilise(scalingVector.y(), pTraits<solveScalar>::vsmall); scalingFactor[0]
/ stabilise(scalingFactor[1], pTraits<solveScalar>::vsmall)
);
if (debug >= 2) if (debug >= 2)
{ {

2
src/Pstream/dummy/Make/files
View File

@ -1,4 +1,6 @@
UPstream.C UPstream.C
UPstreamBroadcast.C
UPstreamReduce.C
UIPstreamRead.C UIPstreamRead.C
UOPstreamWrite.C UOPstreamWrite.C

97
src/Pstream/dummy/UPstream.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2011-2018 OpenFOAM Foundation Copyright (C) 2011-2018 OpenFOAM Foundation
Copyright (C) 2016-2021 OpenCFD Ltd. Copyright (C) 2016-2022 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
@ -27,7 +27,6 @@ License
\*---------------------------------------------------------------------------*/ \*---------------------------------------------------------------------------*/
#include "Pstream.H" #include "Pstream.H"
#include "PstreamReduceOps.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
@ -74,99 +73,7 @@ void Foam::UPstream::abort()
} }
void Foam::reduce(scalar&, const sumOp<scalar>&, const int, const label) // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
{}
void Foam::reduce(scalar&, const minOp<scalar>&, const int, const label)
{}
void Foam::reduce(vector2D&, const sumOp<vector2D>&, const int, const label)
{}
void Foam::sumReduce
(
scalar&,
label&,
const int,
const label
)
{}
void Foam::reduce(scalar&, const sumOp<scalar>&, const int, const label, label&)
{}
void Foam::reduce
(
scalar[],
const int,
const sumOp<scalar>&,
const int,
const label,
label&
)
{}
#if defined(WM_SPDP)
void Foam::reduce
(
solveScalar& Value,
const sumOp<solveScalar>& bop,
const int tag,
const label comm
)
{}
void Foam::reduce
(
solveScalar& Value,
const minOp<solveScalar>& bop,
const int tag,
const label comm
)
{}
void Foam::reduce
(
Vector2D<solveScalar>& Value,
const sumOp<Vector2D<solveScalar>>& bop,
const int tag,
const label comm
)
{}
void Foam::sumReduce
(
solveScalar& Value,
label& Count,
const int tag,
const label comm
)
{}
void Foam::reduce
(
solveScalar& Value,
const sumOp<solveScalar>& bop,
const int tag,
const label comm,
label& request
)
{}
void Foam::reduce
(
solveScalar[],
const int,
const sumOp<solveScalar>&,
const int,
const label,
label&
)
{}
#endif
void Foam::UPstream::allToAll void Foam::UPstream::allToAll
( (

45
src/Pstream/dummy/UPstreamBroadcast.C Normal file
View File

@ -0,0 +1,45 @@
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
\*---------------------------------------------------------------------------*/
#include "UPstream.H"
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
bool Foam::UPstream::broadcast
(
char* buf,
const std::streamsize bufSize,
const label communicator,
const int rootProcNo
)
{
// Nothing to do - ignore
return true;
}
// ************************************************************************* //

153
src/Pstream/dummy/UPstreamReduce.C Normal file
View File

@ -0,0 +1,153 @@
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
\*---------------------------------------------------------------------------*/
#include "Pstream.H"
#include "PstreamReduceOps.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
// Specialisations for bool
void Foam::reduce
(
bool& value,
const andOp<bool>&,
const int tag,
const label comm
)
{}
void Foam::reduce
(
bool& value,
const orOp<bool>&,
const int tag,
const label comm
)
{}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
// Specialisations for common reduction types
#undef Pstream_CommonReductions
#define Pstream_CommonReductions(Native) \
\
void Foam::reduce \
( \
Native& value, \
const minOp<Native>&, \
const int tag, \
const label comm \
) \
{} \
\
void Foam::reduce \
( \
Native& value, \
const maxOp<Native>&, \
const int tag, \
const label comm \
) \
{} \
\
void Foam::reduce \
( \
Native& value, \
const sumOp<Native>&, \
const int tag, \
const label comm \
) \
{} \
\
void Foam::reduce \
( \
Native values[], \
const int size, \
const sumOp<Native>&, \
const int tag, \
const label comm \
) \
{}
Pstream_CommonReductions(int32_t);
Pstream_CommonReductions(int64_t);
Pstream_CommonReductions(uint32_t);
Pstream_CommonReductions(uint64_t);
Pstream_CommonReductions(float);
Pstream_CommonReductions(double);
#undef Pstream_CommonReductions
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
// Specialisations for floating-point types
#undef Pstream_FloatReductions
#define Pstream_FloatReductions(Native) \
\
void Foam::sumReduce \
( \
Native& value, \
label& count, \
const int tag, \
const label comm \
) \
{} \
\
void Foam::reduce \
( \
Native& value, \
const sumOp<Native>&, \
const int tag, \
const label comm, \
label& requestID \
) \
{} \
\
void Foam::reduce \
( \
Native values[], \
const int size, \
const sumOp<Native>&, \
const int tag, \
const label comm, \
label& requestID \
) \
{}
Pstream_FloatReductions(float);
Pstream_FloatReductions(double);
#undef Pstream_FloatReductions
// ************************************************************************* //

2
src/Pstream/mpi/Make/files
View File

@ -1,5 +1,7 @@
PstreamGlobals.C PstreamGlobals.C
UPstream.C UPstream.C
UPstreamBroadcast.C
UPstreamReduce.C
UIPstreamRead.C UIPstreamRead.C
UOPstreamWrite.C UOPstreamWrite.C

8
src/Pstream/mpi/PstreamGlobals.H
View File

@ -35,8 +35,8 @@ SourceFiles
\*---------------------------------------------------------------------------*/ \*---------------------------------------------------------------------------*/
#ifndef PstreamGlobals_H #ifndef Foam_PstreamGlobals_H
#define PstreamGlobals_H #define Foam_PstreamGlobals_H
#include "DynamicList.H" #include "DynamicList.H"
#include <mpi.h> #include <mpi.h>
@ -62,14 +62,12 @@ extern DynamicList<int> freedTags_;
extern DynamicList<MPI_Comm> MPICommunicators_; extern DynamicList<MPI_Comm> MPICommunicators_;
extern DynamicList<MPI_Group> MPIGroups_; extern DynamicList<MPI_Group> MPIGroups_;
void checkCommunicator(const label comm, const label toProcNo); void checkCommunicator(const label comm, const label toProcNo);
};
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
} // End namespace PstreamGlobals
} // End namespace Foam } // End namespace Foam
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //

297
src/Pstream/mpi/UPstream.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2016-2021 OpenCFD Ltd. Copyright (C) 2016-2022 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
@ -40,17 +40,6 @@ License
#include <cstdlib> #include <cstdlib>
#include <csignal> #include <csignal>
#if defined(WM_SP)
#define MPI_SCALAR MPI_FLOAT
#define MPI_SOLVESCALAR MPI_FLOAT
#elif defined(WM_SPDP)
#define MPI_SCALAR MPI_FLOAT
#define MPI_SOLVESCALAR MPI_DOUBLE
#elif defined(WM_DP)
#define MPI_SCALAR MPI_DOUBLE
#define MPI_SOLVESCALAR MPI_DOUBLE
#endif
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * // // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
// The min value and default for MPI buffers length // The min value and default for MPI buffers length
@ -491,245 +480,7 @@ void Foam::UPstream::abort()
} }
void Foam::reduce // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
(
scalar& Value,
const sumOp<scalar>& bop,
const int tag,
const label communicator
)
{
if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
{
Pout<< "** reducing:" << Value << " with comm:" << communicator
<< " warnComm:" << UPstream::warnComm
<< endl;
error::printStack(Pout);
}
allReduce(Value, 1, MPI_SCALAR, MPI_SUM, bop, tag, communicator);
}
void Foam::reduce
(
scalar& Value,
const minOp<scalar>& bop,
const int tag,
const label communicator
)
{
if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
{
Pout<< "** reducing:" << Value << " with comm:" << communicator
<< " warnComm:" << UPstream::warnComm
<< endl;
error::printStack(Pout);
}
allReduce(Value, 1, MPI_SCALAR, MPI_MIN, bop, tag, communicator);
}
void Foam::reduce
(
vector2D& Value,
const sumOp<vector2D>& bop,
const int tag,
const label communicator
)
{
if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
{
Pout<< "** reducing:" << Value << " with comm:" << communicator
<< " warnComm:" << UPstream::warnComm
<< endl;
error::printStack(Pout);
}
allReduce(Value, 2, MPI_SCALAR, MPI_SUM, bop, tag, communicator);
}
void Foam::sumReduce
(
scalar& Value,
label& Count,
const int tag,
const label communicator
)
{
if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
{
Pout<< "** sumReduce:" << Value << " with comm:" << communicator
<< " warnComm:" << UPstream::warnComm
<< endl;
error::printStack(Pout);
}
vector2D twoScalars(Value, scalar(Count));
reduce(twoScalars, sumOp<vector2D>(), tag, communicator);
Value = twoScalars.x();
Count = twoScalars.y();
}
void Foam::reduce
(
scalar& Value,
const sumOp<scalar>& bop,
const int tag,
const label communicator,
label& requestID
)
{
iallReduce<scalar>(&Value, 1, MPI_SCALAR, MPI_SUM, communicator, requestID);
}
void Foam::reduce
(
scalar values[],
const int size,
const sumOp<scalar>& bop,
const int tag,
const label communicator,
label& requestID
)
{
iallReduce<scalar>
(
values,
size,
MPI_SCALAR,
MPI_SUM,
communicator,
requestID
);
}
#if defined(WM_SPDP)
void Foam::reduce
(
solveScalar& Value,
const sumOp<solveScalar>& bop,
const int tag,
const label communicator
)
{
if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
{
Pout<< "** reducing:" << Value << " with comm:" << communicator
<< " warnComm:" << UPstream::warnComm
<< endl;
error::printStack(Pout);
}
allReduce(Value, 1, MPI_SOLVESCALAR, MPI_SUM, bop, tag, communicator);
}
void Foam::reduce
(
solveScalar& Value,
const minOp<solveScalar>& bop,
const int tag,
const label communicator
)
{
if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
{
Pout<< "** reducing:" << Value << " with comm:" << communicator
<< " warnComm:" << UPstream::warnComm
<< endl;
error::printStack(Pout);
}
allReduce(Value, 1, MPI_SOLVESCALAR, MPI_MIN, bop, tag, communicator);
}
void Foam::reduce
(
Vector2D<solveScalar>& Value,
const sumOp<Vector2D<solveScalar>>& bop,
const int tag,
const label communicator
)
{
if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
{
Pout<< "** reducing:" << Value << " with comm:" << communicator
<< " warnComm:" << UPstream::warnComm
<< endl;
error::printStack(Pout);
}
allReduce(Value, 2, MPI_SOLVESCALAR, MPI_SUM, bop, tag, communicator);
}
void Foam::sumReduce
(
solveScalar& Value,
label& Count,
const int tag,
const label communicator
)
{
if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
{
Pout<< "** reducing:" << Value << " with comm:" << communicator
<< " warnComm:" << UPstream::warnComm
<< endl;
error::printStack(Pout);
}
Vector2D<solveScalar> twoScalars(Value, solveScalar(Count));
reduce(twoScalars, sumOp<Vector2D<solveScalar>>(), tag, communicator);
Value = twoScalars.x();
Count = twoScalars.y();
}
void Foam::reduce
(
solveScalar& Value,
const sumOp<solveScalar>& bop,
const int tag,
const label communicator,
label& requestID
)
{
iallReduce<solveScalar>
(
&Value,
1,
MPI_SOLVESCALAR,
MPI_SUM,
communicator,
requestID
);
}
void Foam::reduce
(
solveScalar values[],
const int size,
const sumOp<solveScalar>& bop,
const int tag,
const label communicator,
label& requestID
)
{
iallReduce<solveScalar>
(
values,
size,
MPI_SOLVESCALAR,
MPI_SUM,
communicator,
requestID
);
}
#endif
void Foam::UPstream::allToAll void Foam::UPstream::allToAll
( (
@ -1437,22 +1188,15 @@ int Foam::UPstream::allocateTag(const char* s)
if (debug) if (debug)
{ {
//if (UPstream::lateBlocking > 0) Pout<< "UPstream::allocateTag "
//{ << s << " : tag:" << tag << endl;
// string& poutp = Pout.prefix();
// poutp[poutp.size()-2*(UPstream::lateBlocking+2)+tag] = 'X';
// Perr.prefix() = Pout.prefix();
//}
Pout<< "UPstream::allocateTag " << s
<< " : tag:" << tag
<< endl;
} }
return tag; return tag;
} }
int Foam::UPstream::allocateTag(const word& s) int Foam::UPstream::allocateTag(const std::string& s)
{ {
int tag; int tag;
if (PstreamGlobals::freedTags_.size()) if (PstreamGlobals::freedTags_.size())
@ -1466,15 +1210,8 @@ int Foam::UPstream::allocateTag(const word& s)
if (debug) if (debug)
{ {
//if (UPstream::lateBlocking > 0) Pout<< "UPstream::allocateTag "
//{ << s.c_str() << " : tag:" << tag << endl;
// string& poutp = Pout.prefix();
// poutp[poutp.size()-2*(UPstream::lateBlocking+2)+tag] = 'X';
// Perr.prefix() = Pout.prefix();
//}
Pout<< "UPstream::allocateTag " << s
<< " : tag:" << tag
<< endl;
} }
return tag; return tag;
@ -1485,29 +1222,19 @@ void Foam::UPstream::freeTag(const char* s, const int tag)
{ {
if (debug) if (debug)
{ {
//if (UPstream::lateBlocking > 0) Pout<< "UPstream::freeTag "
//{ << s << " tag:" << tag << endl;
// string& poutp = Pout.prefix();
// poutp[poutp.size()-2*(UPstream::lateBlocking+2)+tag] = ' ';
// Perr.prefix() = Pout.prefix();
//}
Pout<< "UPstream::freeTag " << s << " tag:" << tag << endl;
} }
PstreamGlobals::freedTags_.append(tag); PstreamGlobals::freedTags_.append(tag);
} }
void Foam::UPstream::freeTag(const word& s, const int tag) void Foam::UPstream::freeTag(const std::string& s, const int tag)
{ {
if (debug) if (debug)
{ {
//if (UPstream::lateBlocking > 0) Pout<< "UPstream::freeTag "
//{ << s.c_str() << " tag:" << tag << endl;
// string& poutp = Pout.prefix();
// poutp[poutp.size()-2*(UPstream::lateBlocking+2)+tag] = ' ';
// Perr.prefix() = Pout.prefix();
//}
Pout<< "UPstream::freeTag " << s << " tag:" << tag << endl;
} }
PstreamGlobals::freedTags_.append(tag); PstreamGlobals::freedTags_.append(tag);
} }

93
src/Pstream/mpi/UPstreamBroadcast.C Normal file
View File

@ -0,0 +1,93 @@
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
\*---------------------------------------------------------------------------*/
#include "UPstream.H"
#include "PstreamGlobals.H"
#include "profilingPstream.H"
#include <mpi.h>
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
bool Foam::UPstream::broadcast
(
char* buf,
const std::streamsize bufSize,
const label communicator,
const int rootProcNo
)
{
if (!UPstream::parRun() || UPstream::nProcs(communicator) < 2)
{
// Nothing to do - ignore
return true;
}
//Needed? PstreamGlobals::checkCommunicator(communicator, rootProcNo);
if (debug)
{
Pout<< "UPstream::broadcast : root:" << rootProcNo
<< " comm:" << communicator
<< " size:" << label(bufSize)
<< Foam::endl;
}
if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
{
Pout<< "UPstream::broadcast : root:" << rootProcNo
<< " comm:" << communicator
<< " size:" << label(bufSize)
<< " warnComm:" << UPstream::warnComm
<< Foam::endl;
error::printStack(Pout);
}
profilingPstream::beginTiming();
bool failed = MPI_Bcast
(
buf,
bufSize,
MPI_BYTE,
rootProcNo,
PstreamGlobals::MPICommunicators_[communicator]
);
if (rootProcNo == UPstream::myProcNo(communicator))
{
profilingPstream::addScatterTime();
}
else
{
profilingPstream::addGatherTime();
}
return !failed;
}
// ************************************************************************* //

211
src/Pstream/mpi/UPstreamReduce.C Normal file
View File

@ -0,0 +1,211 @@
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
\*---------------------------------------------------------------------------*/
#include "Pstream.H"
#include "PstreamReduceOps.H"
#include "allReduce.H"
#include <mpi.h>
#include <cinttypes>
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
// Specialisations for bool
void Foam::reduce
(
bool& value,
const andOp<bool>&,
const int tag,
const label comm
)
{
// This can also work:
// PstreamDetail::allReduce(&value, 1, MPI_BYTE, MPI_BAND, tag, comm);
PstreamDetail::allReduce(&value, 1, MPI_C_BOOL, MPI_LAND, tag, comm);
}
void Foam::reduce
(
bool& value,
const orOp<bool>&,
const int tag,
const label comm
)
{
// This can also work:
// PstreamDetail::allReduce(&value, 1, MPI_BYTE, MPI_BOR, tag, comm);
PstreamDetail::allReduce(&value, 1, MPI_C_BOOL, MPI_LOR, tag, comm);
}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
// Specialisations for common reduction types
#undef Pstream_CommonReductions
#define Pstream_CommonReductions(Native, TaggedType) \
\
void Foam::reduce \
( \
Native& value, \
const minOp<Native>&, \
const int tag, \
const label comm \
) \
{ \
PstreamDetail::allReduce<Native> \
( \
&value, 1, TaggedType, MPI_MIN, tag, comm \
); \
} \
\
void Foam::reduce \
( \
Native& value, \
const maxOp<Native>&, \
const int tag, \
const label comm \
) \
{ \
PstreamDetail::allReduce<Native> \
( \
&value, 1, TaggedType, MPI_MAX, tag, comm \
); \
} \
\
void Foam::reduce \
( \
Native& value, \
const sumOp<Native>&, \
const int tag, \
const label comm \
) \
{ \
PstreamDetail::allReduce<Native> \
( \
&value, 1, TaggedType, MPI_SUM, tag, comm \
); \
} \
\
void Foam::reduce \
( \
Native values[], \
const int size, \
const sumOp<Native>&, \
const int tag, \
const label comm \
) \
{ \
PstreamDetail::allReduce<Native> \
( \
values, size, TaggedType, MPI_SUM, tag, comm \
); \
} \
Pstream_CommonReductions(int32_t, MPI_INT32_T);
Pstream_CommonReductions(int64_t, MPI_INT64_T);
Pstream_CommonReductions(uint32_t, MPI_UINT32_T);
Pstream_CommonReductions(uint64_t, MPI_UINT64_T);
Pstream_CommonReductions(float, MPI_FLOAT);
Pstream_CommonReductions(double, MPI_DOUBLE);
#undef Pstream_CommonReductions
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
// Specialisations for floating-point types
#undef Pstream_FloatReductions
#define Pstream_FloatReductions(Native, TaggedType) \
\
void Foam::sumReduce \
( \
Native& value, \
label& count, \
const int tag, \
const label comm \
) \
{ \
if (UPstream::parRun()) \
{ \
Native values[2]; \
values[0] = value; \
values[1] = static_cast<Native>(count); \
\
PstreamDetail::allReduce<Native> \
( \
values, 2, TaggedType, MPI_SUM, tag, comm \
); \
\
value = values[0]; \
count = static_cast<label>(values[1]); \
} \
} \
\
void Foam::reduce \
( \
Native& value, \
const sumOp<Native>&, \
const int tag, \
const label comm, \
label& requestID \
) \
{ \
PstreamDetail::iallReduce<Native> \
( \
&value, 1, TaggedType, MPI_SUM, comm, requestID \
); \
} \
\
void Foam::reduce \
( \
Native values[], \
const int size, \
const sumOp<Native>&, \
const int tag, \
const label comm, \
label& requestID \
) \
{ \
PstreamDetail::iallReduce<Native> \
( \
values, size, TaggedType, MPI_SUM, comm, requestID \
); \
}
Pstream_FloatReductions(float, MPI_FLOAT);
Pstream_FloatReductions(double, MPI_DOUBLE);
#undef Pstream_FloatReductions
// ************************************************************************* //

45
src/Pstream/mpi/allReduce.H
View File

@ -6,6 +6,7 @@
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2012-2016 OpenFOAM Foundation Copyright (C) 2012-2016 OpenFOAM Foundation
Copyright (C) 2022 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
@ -23,37 +24,52 @@ License
You should have received a copy of the GNU General Public License You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>. along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
InNamespace Namespace
Foam Foam::PstreamDetail
Description Description
Various functions to wrap MPI_Allreduce Some implementation details for Pstream and/or MPI.
InNamespace
Foam::PstreamDetail
Description
Functions to wrap MPI_Bcast, MPI_Allreduce, MPI_Iallreduce
SourceFiles SourceFiles
allReduceTemplates.C allReduceTemplates.C
\*---------------------------------------------------------------------------*/ \*---------------------------------------------------------------------------*/
#ifndef allReduce_H #ifndef Foam_allReduce_H
#define allReduce_H #define Foam_allReduce_H
#include "UPstream.H" #include "UPstream.H"
#include <mpi.h> #include <mpi.h>
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
namespace Foam namespace Foam
{ {
namespace PstreamDetail
{
template<class Type, class BinaryOp> template<class Type>
void allBroadcast
(
Type* values,
int count,
MPI_Datatype datatype,
const label communicator
);
template<class Type>
void allReduce void allReduce
( (
Type& Value, Type* values,
int count, int count,
MPI_Datatype MPIType, MPI_Datatype datatype,
MPI_Op op, MPI_Op optype,
const BinaryOp& bop,
const int tag, const int tag,
const label communicator const label communicator
); );
@ -61,16 +77,17 @@ void allReduce
template<class Type> template<class Type>
void iallReduce void iallReduce
( (
void* Value, Type* values,
int count, int count,
MPI_Datatype MPIType, MPI_Datatype datatype,
MPI_Op op, MPI_Op optype,
const label communicator, const label communicator,
label& requestID label& requestID
); );
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
} // End namespace PstreamDetail
} // End namespace Foam } // End namespace Foam
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //

220
src/Pstream/mpi/allReduceTemplates.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2012-2015 OpenFOAM Foundation Copyright (C) 2012-2015 OpenFOAM Foundation
Copyright (C) 2019-2020 OpenCFD Ltd. Copyright (C) 2019-2022 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
@ -28,18 +28,16 @@ License
#include "allReduce.H" #include "allReduce.H"
#include "profilingPstream.H" #include "profilingPstream.H"
#include "PstreamGlobals.H"
// * * * * * * * * * * * * * * * Global Functions * * * * * * * * * * * * * // // * * * * * * * * * * * * * * * Global Functions * * * * * * * * * * * * * //
template<class Type, class BinaryOp> template<class Type>
void Foam::allReduce void Foam::PstreamDetail::allBroadcast
( (
Type& Value, Type* values,
int MPICount, int count,
MPI_Datatype MPIType, MPI_Datatype datatype,
MPI_Op MPIOp,
const BinaryOp& bop,
const int tag,
const label communicator const label communicator
) )
{ {
@ -50,129 +48,76 @@ void Foam::allReduce
profilingPstream::beginTiming(); profilingPstream::beginTiming();
if (UPstream::nProcs(communicator) <= UPstream::nProcsSimpleSum) // const int retval =
{ MPI_Bcast
if (UPstream::master(communicator))
{
for (const int proci : UPstream::subProcs(communicator))
{
Type value;
if
( (
MPI_Recv values,
( count,
&value, datatype,
MPICount, 0, // (root process) is master == UPstream::masterNo()
MPIType,
proci,
tag,
PstreamGlobals::MPICommunicators_[communicator],
MPI_STATUS_IGNORE
)
)
{
FatalErrorInFunction
<< "MPI_Recv failed"
<< Foam::abort(FatalError);
}
Value = bop(Value, value);
}
}
else
{
if
(
MPI_Send
(
&Value,
MPICount,
MPIType,
UPstream::masterNo(),
tag,
PstreamGlobals::MPICommunicators_[communicator]
)
)
{
FatalErrorInFunction
<< "MPI_Send failed"
<< Foam::abort(FatalError);
}
}
if (UPstream::master(communicator))
{
for (const int proci : UPstream::subProcs(communicator))
{
if
(
MPI_Send
(
&Value,
MPICount,
MPIType,
proci,
tag,
PstreamGlobals::MPICommunicators_[communicator]
)
)
{
FatalErrorInFunction
<< "MPI_Send failed"
<< Foam::abort(FatalError);
}
}
}
else
{
if
(
MPI_Recv
(
&Value,
MPICount,
MPIType,
UPstream::masterNo(),
tag,
PstreamGlobals::MPICommunicators_[communicator],
MPI_STATUS_IGNORE
)
)
{
FatalErrorInFunction
<< "MPI_Recv failed"
<< Foam::abort(FatalError);
}
}
}
else
{
Type sum;
MPI_Allreduce
(
&Value,
&sum,
MPICount,
MPIType,
MPIOp,
PstreamGlobals::MPICommunicators_[communicator] PstreamGlobals::MPICommunicators_[communicator]
); );
Value = sum;
profilingPstream::addScatterTime();
} }
template<class Type>
void Foam::PstreamDetail::allReduce
(
Type* values,
int count,
MPI_Datatype datatype,
MPI_Op optype,
const int tag,
const label communicator
)
{
if (!UPstream::parRun())
{
return;
}
if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
{
Pout<< "** reducing:";
if (count == 1)
{
Pout<< (*values);
}
else
{
Pout<< UList<Type>(values, count);
}
Pout<< " with comm:" << communicator
<< " warnComm:" << UPstream::warnComm << endl;
error::printStack(Pout);
}
profilingPstream::beginTiming();
// const int retval =
MPI_Allreduce
(
MPI_IN_PLACE,
values,
count,
datatype,
optype,
PstreamGlobals::MPICommunicators_[communicator]
);
profilingPstream::addReduceTime(); profilingPstream::addReduceTime();
} }
template<class Type> template<class Type>
void Foam::iallReduce void Foam::PstreamDetail::iallReduce
( (
void* recvBuf, Type* values,
int MPICount, int count,
MPI_Datatype MPIType, MPI_Datatype datatype,
MPI_Op MPIOp, MPI_Op optype,
const label communicator, const label communicator,
label& requestID label& requestID
) )
@ -184,9 +129,16 @@ void Foam::iallReduce
if (UPstream::warnComm != -1 && communicator != UPstream::warnComm) if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
{ {
Pout<< "** non-blocking reducing:" Pout<< "** non-blocking reducing:";
<< UList<Type>(static_cast<Type*>(recvBuf), MPICount) if (count == 1)
<< " with comm:" << communicator {
Pout<< (*values);
}
else
{
Pout<< UList<Type>(values, count);
}
Pout<< " with comm:" << communicator
<< " warnComm:" << UPstream::warnComm << endl; << " warnComm:" << UPstream::warnComm << endl;
error::printStack(Pout); error::printStack(Pout);
} }
@ -200,10 +152,10 @@ void Foam::iallReduce
MPI_Iallreduce MPI_Iallreduce
( (
MPI_IN_PLACE, MPI_IN_PLACE,
recvBuf, values,
MPICount, count,
MPIType, datatype,
MPIOp, optype,
PstreamGlobals::MPICommunicators_[communicator], PstreamGlobals::MPICommunicators_[communicator],
&request &request
) )
@ -211,7 +163,7 @@ void Foam::iallReduce
{ {
FatalErrorInFunction FatalErrorInFunction
<< "MPI_Iallreduce failed for " << "MPI_Iallreduce failed for "
<< UList<Type>(static_cast<Type*>(recvBuf), MPICount) << UList<Type>(values, count)
<< Foam::abort(FatalError); << Foam::abort(FatalError);
} }
@ -238,17 +190,17 @@ void Foam::iallReduce
MPI_Allreduce MPI_Allreduce
( (
MPI_IN_PLACE, MPI_IN_PLACE,
recvBuf, values,
MPICount, count,
MPIType, datatype,
MPIOp, optype,
PstreamGlobals::MPICommunicators_[communicator] PstreamGlobals::MPICommunicators_[communicator]
) )
) )
{ {
FatalErrorInFunction FatalErrorInFunction
<< "MPI_Allreduce failed for " << "MPI_Allreduce failed for "
<< UList<Type>(static_cast<Type*>(recvBuf), MPICount) << UList<Type>(values, count)
<< Foam::abort(FatalError); << Foam::abort(FatalError);
} }
requestID = -1; requestID = -1;

33
src/fvOptions/sources/derived/meanVelocityForce/patchMeanVelocityForce/patchMeanVelocityForce.C
View File

@ -6,6 +6,7 @@
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2015-2016 OpenFOAM Foundation Copyright (C) 2015-2016 OpenFOAM Foundation
Copyright (C) 2022 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
@ -71,15 +72,15 @@ Foam::scalar Foam::fv::patchMeanVelocityForce::magUbarAve
const volVectorField& U const volVectorField& U
) const ) const
{ {
vector2D sumAmagUsumA FixedList<scalar, 2> sumAmagUsumA(Zero);
(
sumAmagUsumA[0] +=
sum sum
( (
(flowDir_ & U.boundaryField()[patchi_]) (flowDir_ & U.boundaryField()[patchi_])
* mesh_.boundary()[patchi_].magSf() * mesh_.boundary()[patchi_].magSf()
),
sum(mesh_.boundary()[patchi_].magSf())
); );
sumAmagUsumA[1] += sum(mesh_.boundary()[patchi_].magSf());
// If the mean velocity force is applied to a cyclic patch // If the mean velocity force is applied to a cyclic patch
@ -89,29 +90,29 @@ Foam::scalar Foam::fv::patchMeanVelocityForce::magUbarAve
if (Pstream::parRun() && isA<cyclicPolyPatch>(patches[patchi_])) if (Pstream::parRun() && isA<cyclicPolyPatch>(patches[patchi_]))
{ {
labelList processorCyclicPatches for
( (
processorCyclicPolyPatch::patchIDs(patch_, patches) const label patchi
); : processorCyclicPolyPatch::patchIDs(patch_, patches)
)
forAll(processorCyclicPatches, pcpi)
{ {
const label patchi = processorCyclicPatches[pcpi]; sumAmagUsumA[0] +=
sumAmagUsumA.x() +=
sum sum
( (
(flowDir_ & U.boundaryField()[patchi]) (flowDir_ & U.boundaryField()[patchi])
* mesh_.boundary()[patchi].magSf() * mesh_.boundary()[patchi].magSf()
); );
sumAmagUsumA[1] += sum(mesh_.boundary()[patchi].magSf());
sumAmagUsumA.y() += sum(mesh_.boundary()[patchi].magSf());
} }
} }
mesh_.reduce(sumAmagUsumA, sumOp<vector2D>()); mesh_.reduce(sumAmagUsumA, sumOp<scalar>());
return sumAmagUsumA.x()/sumAmagUsumA.y(); return
(
sumAmagUsumA[0]
/ stabilise(sumAmagUsumA[1], VSMALL)
);
} }

10
src/lumpedPointMotion/state/lumpedPointState.C
View File

@ -5,7 +5,7 @@
\\ / A nd | www.openfoam.com \\ / A nd | www.openfoam.com
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2016-2020 OpenCFD Ltd. Copyright (C) 2016-2022 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
@ -405,14 +405,8 @@ bool Foam::lumpedPointState::readData
{ {
// Scatter master data using communication scheme // Scatter master data using communication scheme
const List<Pstream::commsStruct>& comms =
(
(Pstream::nProcs() < Pstream::nProcsSimpleSum)
? Pstream::linearCommunication()
: Pstream::treeCommunication()
);
// Get my communication order // Get my communication order
const List<Pstream::commsStruct>& comms = Pstream::whichCommunication();
const Pstream::commsStruct& myComm = comms[Pstream::myProcNo()]; const Pstream::commsStruct& myComm = comms[Pstream::myProcNo()];
// Receive from up // Receive from up