mirror of
https://develop.openfoam.com/Development/openfoam.git
synced 2025-11-28 03:28:01 +00:00
ENH: update globalIndex::mpiGather to use MPI intrinsic/user types
- add provisional support for selecting MPI_Gatherv when merging fields in the surface writers. Uses the 'gatherv' keyword [experimental] BUG: gatherv/scatterv wrappers used the incorrect data type - incorrectly checked against UPstream_basic_dataType trait instead of UPstream_dataType, which resulted in a count mismatch for user-defined types (eg, vector, tensor,...). This was not visibly active bug, since previous internal uses of gatherv were restricted to primitive types. COMP: make UPstream dataType traits size(...) constexpr - allows static asserts and/or compile-time selection of code based on size(1)
This commit is contained in:
@ -200,7 +200,7 @@ void printTypeName()
|
|||||||
|
|
||||||
|
|
||||||
template<class Type, bool UseTypeName = true>
|
template<class Type, bool UseTypeName = true>
|
||||||
void printPstreamTraits(const std::string_view name = std::string_view())
|
void printPstreamTraits(std::string_view name = std::string_view())
|
||||||
{
|
{
|
||||||
Info<< "========" << nl;
|
Info<< "========" << nl;
|
||||||
Info<< "type: ";
|
Info<< "type: ";
|
||||||
@ -299,6 +299,9 @@ void printPstreamTraits(const std::string_view name = std::string_view())
|
|||||||
// Use element or component type (or byte-wise) for data type
|
// Use element or component type (or byte-wise) for data type
|
||||||
using base = typename UPstream_dataType<Type>::base;
|
using base = typename UPstream_dataType<Type>::base;
|
||||||
|
|
||||||
|
// The sizing factor is constexpr
|
||||||
|
constexpr std::streamsize count = UPstream_dataType<Type>::size(1);
|
||||||
|
|
||||||
Info<< " : ";
|
Info<< " : ";
|
||||||
if constexpr (UseTypeName)
|
if constexpr (UseTypeName)
|
||||||
{
|
{
|
||||||
@ -311,8 +314,7 @@ void printPstreamTraits(const std::string_view name = std::string_view())
|
|||||||
|
|
||||||
Info<< " cmpt-type=";
|
Info<< " cmpt-type=";
|
||||||
printDataTypeId(UPstream_dataType<Type>::datatype_id);
|
printDataTypeId(UPstream_dataType<Type>::datatype_id);
|
||||||
Info<< " count=" << UPstream_dataType<Type>::size(1);
|
Info<< " count=" << count << nl;
|
||||||
Info<< nl;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -362,6 +364,24 @@ void print_data_opType(BinaryOp bop, std::string_view name)
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
template<class Type>
|
||||||
|
int check_simple(std::string_view name = std::string_view())
|
||||||
|
{
|
||||||
|
// The sizing factor is constexpr
|
||||||
|
constexpr std::streamsize count = UPstream_dataType<Type>::size(1);
|
||||||
|
|
||||||
|
static_assert
|
||||||
|
(
|
||||||
|
(count == 1),
|
||||||
|
"Code does not (yet) work with aggregate types"
|
||||||
|
);
|
||||||
|
|
||||||
|
Info<< "check_simple: " << name << ": " << count << nl;
|
||||||
|
return count;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
|
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
|
||||||
// Main program:
|
// Main program:
|
||||||
|
|
||||||
@ -389,6 +409,8 @@ int main()
|
|||||||
printPstreamTraits<const float>();
|
printPstreamTraits<const float>();
|
||||||
printPstreamTraits<floatVector>();
|
printPstreamTraits<floatVector>();
|
||||||
|
|
||||||
|
check_simple<floatVector>("vector<float>");
|
||||||
|
|
||||||
printPstreamTraits<scalar>();
|
printPstreamTraits<scalar>();
|
||||||
printPstreamTraits<double>();
|
printPstreamTraits<double>();
|
||||||
printPstreamTraits<doubleVector>();
|
printPstreamTraits<doubleVector>();
|
||||||
|
|||||||
@ -1,4 +1,4 @@
|
|||||||
mydebugSurfaceWriter.C
|
mydebugSurfaceWriter.cxx
|
||||||
Test-surface-sampling.C
|
Test-surface-sampling.cxx
|
||||||
|
|
||||||
EXE = $(FOAM_USER_APPBIN)/Test-surface-sampling
|
EXE = $(FOAM_USER_APPBIN)/Test-surface-sampling
|
||||||
|
|||||||
@ -5,7 +5,7 @@
|
|||||||
\\ / A nd | www.openfoam.com
|
\\ / A nd | www.openfoam.com
|
||||||
\\/ M anipulation |
|
\\/ M anipulation |
|
||||||
-------------------------------------------------------------------------------
|
-------------------------------------------------------------------------------
|
||||||
Copyright (C) 2022-2023 OpenCFD Ltd.
|
Copyright (C) 2022-2025 OpenCFD Ltd.
|
||||||
-------------------------------------------------------------------------------
|
-------------------------------------------------------------------------------
|
||||||
License
|
License
|
||||||
This file is part of OpenFOAM.
|
This file is part of OpenFOAM.
|
||||||
@ -83,12 +83,10 @@ template<> struct narrowType<SymmTensor<double>>
|
|||||||
typedef SymmTensor<float> type;
|
typedef SymmTensor<float> type;
|
||||||
};
|
};
|
||||||
|
|
||||||
// FIXME: Not sure why this one seems to be broken...
|
template<> struct narrowType<Tensor<double>>
|
||||||
//
|
{
|
||||||
// template<> struct narrowType<Tensor<double>>
|
typedef Tensor<float> type;
|
||||||
// {
|
};
|
||||||
// typedef Tensor<float> type;
|
|
||||||
// };
|
|
||||||
|
|
||||||
} // End namespace Foam
|
} // End namespace Foam
|
||||||
|
|
||||||
@ -104,12 +102,18 @@ Foam::surfaceWriters::mydebugWriter::mergeField
|
|||||||
{
|
{
|
||||||
addProfiling(merge, "debugWriter::merge-field");
|
addProfiling(merge, "debugWriter::merge-field");
|
||||||
|
|
||||||
// This is largely identical to surfaceWriter::mergeField()
|
// Identical to surfaceWriter::mergeField()
|
||||||
// but with narrowing for communication
|
// but with narrowing for communication
|
||||||
if (narrowTransfer_ && parallel_ && UPstream::parRun())
|
|
||||||
|
if constexpr (std::is_same_v<Tensor<double>, Type>)
|
||||||
|
{
|
||||||
|
// Cannot narrow tensor. Does not compile since MatrixSpace
|
||||||
|
// does not (yet) allow assigments from different Cmpt types.
|
||||||
|
}
|
||||||
|
else if (narrowTransfer_ && parallel_ && UPstream::parRun())
|
||||||
{
|
{
|
||||||
// The narrowed type
|
// The narrowed type
|
||||||
typedef typename narrowType<Type>::type narrowedType;
|
using narrowedType = typename narrowType<Type>::type;
|
||||||
|
|
||||||
// Ensure geometry is also merged
|
// Ensure geometry is also merged
|
||||||
merge();
|
merge();
|
||||||
@ -130,14 +134,29 @@ Foam::surfaceWriters::mydebugWriter::mergeField
|
|||||||
ConstPrecisionAdaptor<narrowedType, Type> input(fld);
|
ConstPrecisionAdaptor<narrowedType, Type> input(fld);
|
||||||
PrecisionAdaptor<narrowedType, Type> output(allFld);
|
PrecisionAdaptor<narrowedType, Type> output(allFld);
|
||||||
|
|
||||||
globIndex.gather
|
if (gatherv_)
|
||||||
(
|
{
|
||||||
input.cref(), // fld,
|
globIndex.mpiGather
|
||||||
output.ref(), // allFld,
|
(
|
||||||
UPstream::msgType(),
|
input.cref(), // fld
|
||||||
commType_,
|
output.ref(), // allFld
|
||||||
UPstream::worldComm
|
UPstream::worldComm,
|
||||||
);
|
// For fallback:
|
||||||
|
commType_,
|
||||||
|
UPstream::msgType()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
globIndex.gather
|
||||||
|
(
|
||||||
|
input.cref(), // fld
|
||||||
|
output.ref(), // allFld
|
||||||
|
UPstream::msgType(),
|
||||||
|
commType_,
|
||||||
|
UPstream::worldComm
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
// Commit adapted content changes
|
// Commit adapted content changes
|
||||||
input.commit();
|
input.commit();
|
||||||
@ -193,8 +212,19 @@ Foam::surfaceWriters::mydebugWriter::mydebugWriter
|
|||||||
{
|
{
|
||||||
Info<< "Using debug surface writer ("
|
Info<< "Using debug surface writer ("
|
||||||
<< (this->isPointData() ? "point" : "face") << " data):"
|
<< (this->isPointData() ? "point" : "face") << " data):"
|
||||||
<< " commsType=" << UPstream::commsTypeNames[commType_]
|
<< " commsType=";
|
||||||
<< " merge=" << Switch::name(enableMerge_)
|
|
||||||
|
if (UPstream::parRun())
|
||||||
|
{
|
||||||
|
if (gatherv_) Info<< "gatherv+";
|
||||||
|
Info<< UPstream::commsTypeNames[commType_];
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
Info<< "serial";
|
||||||
|
}
|
||||||
|
|
||||||
|
Info<< " merge=" << Switch::name(enableMerge_)
|
||||||
<< " write=" << Switch::name(enableWrite_)
|
<< " write=" << Switch::name(enableWrite_)
|
||||||
<< " narrow=" << Switch::name(narrowTransfer_)
|
<< " narrow=" << Switch::name(narrowTransfer_)
|
||||||
<< endl;
|
<< endl;
|
||||||
@ -343,11 +343,22 @@ void Foam::UPstream::mpiGatherv
|
|||||||
}
|
}
|
||||||
// Nothing further to do
|
// Nothing further to do
|
||||||
}
|
}
|
||||||
else if constexpr (UPstream_basic_dataType<Type>::value)
|
else if constexpr (UPstream_dataType<Type>::value)
|
||||||
{
|
{
|
||||||
// Restrict to basic (or aliased) MPI types to avoid recalculating
|
// Restrict to basic (or aliased) MPI types to avoid recalculating
|
||||||
// the list of counts/offsets.
|
// the list of counts/offsets.
|
||||||
|
|
||||||
|
// The sizing factor (constexpr) must be 1 otherwise
|
||||||
|
// [recvCounts,recvOffsets] are likely incorrect
|
||||||
|
|
||||||
|
constexpr std::streamsize count = UPstream_dataType<Type>::size(1);
|
||||||
|
|
||||||
|
static_assert
|
||||||
|
(
|
||||||
|
(count == 1),
|
||||||
|
"Code does not (yet) work with aggregate types"
|
||||||
|
);
|
||||||
|
|
||||||
UPstream::mpi_gatherv
|
UPstream::mpi_gatherv
|
||||||
(
|
(
|
||||||
sendData,
|
sendData,
|
||||||
@ -356,7 +367,7 @@ void Foam::UPstream::mpiGatherv
|
|||||||
recvCounts,
|
recvCounts,
|
||||||
recvOffsets,
|
recvOffsets,
|
||||||
|
|
||||||
UPstream_basic_dataType<Type>::datatype_id,
|
UPstream_dataType<Type>::datatype_id,
|
||||||
communicator
|
communicator
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@ -364,7 +375,8 @@ void Foam::UPstream::mpiGatherv
|
|||||||
{
|
{
|
||||||
static_assert
|
static_assert
|
||||||
(
|
(
|
||||||
stdFoam::dependent_false_v<Type>, "Only basic MPI data types"
|
stdFoam::dependent_false_v<Type>,
|
||||||
|
"Only basic and user data types"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -392,11 +404,22 @@ void Foam::UPstream::mpiScatterv
|
|||||||
}
|
}
|
||||||
// Nothing further to do
|
// Nothing further to do
|
||||||
}
|
}
|
||||||
else if constexpr (UPstream_basic_dataType<Type>::value)
|
else if constexpr (UPstream_dataType<Type>::value)
|
||||||
{
|
{
|
||||||
// Restrict to basic (or aliased) MPI types to avoid recalculating
|
// Restrict to basic (or aliased) MPI types to avoid recalculating
|
||||||
// the list of counts/offsets.
|
// the list of counts/offsets.
|
||||||
|
|
||||||
|
// The sizing factor (constexpr) must be 1 otherwise
|
||||||
|
// [sendCounts,sendOffsets] are likely incorrect
|
||||||
|
|
||||||
|
constexpr std::streamsize count = UPstream_dataType<Type>::size(1);
|
||||||
|
|
||||||
|
static_assert
|
||||||
|
(
|
||||||
|
(count == 1),
|
||||||
|
"Code does not (yet) work with aggregate types"
|
||||||
|
);
|
||||||
|
|
||||||
UPstream::mpi_scatterv
|
UPstream::mpi_scatterv
|
||||||
(
|
(
|
||||||
sendData,
|
sendData,
|
||||||
@ -405,7 +428,7 @@ void Foam::UPstream::mpiScatterv
|
|||||||
recvData,
|
recvData,
|
||||||
recvCount,
|
recvCount,
|
||||||
|
|
||||||
UPstream_basic_dataType<Type>::datatype_id,
|
UPstream_dataType<Type>::datatype_id,
|
||||||
communicator
|
communicator
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@ -413,7 +436,8 @@ void Foam::UPstream::mpiScatterv
|
|||||||
{
|
{
|
||||||
static_assert
|
static_assert
|
||||||
(
|
(
|
||||||
stdFoam::dependent_false_v<Type>, "Only basic MPI data types"
|
stdFoam::dependent_false_v<Type>,
|
||||||
|
"Only basic and user data types"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -354,7 +354,7 @@ struct UPstream_basic_dataType
|
|||||||
UPstream_alias_dataType<base>::datatype_id;
|
UPstream_alias_dataType<base>::datatype_id;
|
||||||
|
|
||||||
//- The size in terms of the number of underlying data elements
|
//- The size in terms of the number of underlying data elements
|
||||||
static std::streamsize size(std::streamsize n) noexcept
|
static constexpr std::streamsize size(std::streamsize n) noexcept
|
||||||
{
|
{
|
||||||
if constexpr (UPstream_alias_dataType<T>::value)
|
if constexpr (UPstream_alias_dataType<T>::value)
|
||||||
{
|
{
|
||||||
@ -373,7 +373,10 @@ struct UPstream_basic_dataType
|
|||||||
template<> struct UPstream_basic_dataType<void> : UPstream_mpi_dataType<void>
|
template<> struct UPstream_basic_dataType<void> : UPstream_mpi_dataType<void>
|
||||||
{
|
{
|
||||||
using base = void;
|
using base = void;
|
||||||
static std::streamsize size(std::streamsize n) noexcept { return n; }
|
static constexpr std::streamsize size(std::streamsize n) noexcept
|
||||||
|
{
|
||||||
|
return n;
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
@ -410,7 +413,7 @@ struct UPstream_dataType
|
|||||||
UPstream_any_dataType<base>::datatype_id;
|
UPstream_any_dataType<base>::datatype_id;
|
||||||
|
|
||||||
//- The size in terms of the number of base data elements
|
//- The size in terms of the number of base data elements
|
||||||
static std::streamsize size(std::streamsize n) noexcept
|
static constexpr std::streamsize size(std::streamsize n) noexcept
|
||||||
{
|
{
|
||||||
if constexpr (UPstream_any_dataType<T>::value)
|
if constexpr (UPstream_any_dataType<T>::value)
|
||||||
{
|
{
|
||||||
|
|||||||
@ -27,6 +27,7 @@ License
|
|||||||
\*---------------------------------------------------------------------------*/
|
\*---------------------------------------------------------------------------*/
|
||||||
|
|
||||||
#include "globalIndex.H"
|
#include "globalIndex.H"
|
||||||
|
#include <functional>
|
||||||
|
|
||||||
// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * //
|
// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * //
|
||||||
|
|
||||||
@ -696,159 +697,94 @@ void Foam::globalIndex::mpiGather
|
|||||||
const UList<Type>& sendData,
|
const UList<Type>& sendData,
|
||||||
OutputContainer& allData,
|
OutputContainer& allData,
|
||||||
const label comm,
|
const label comm,
|
||||||
UPstream::commsTypes commsType,
|
[[maybe_unused]] UPstream::commsTypes commsType,
|
||||||
const int tag
|
[[maybe_unused]] const int tag
|
||||||
) const
|
) const
|
||||||
{
|
{
|
||||||
if (!UPstream::parRun())
|
if (!UPstream::is_parallel(comm))
|
||||||
{
|
{
|
||||||
// Serial: direct copy
|
// Serial: direct copy
|
||||||
allData = sendData;
|
allData = sendData;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// MPI_Gatherv requires contiguous data, but a byte-wise transfer can
|
if (UPstream::master(comm))
|
||||||
// quickly exceed the 'int' limits used for MPI sizes/offsets.
|
|
||||||
// Thus gather label/scalar components when possible to increase the
|
|
||||||
// effective size limit.
|
|
||||||
//
|
|
||||||
// Note: cannot rely on pTraits (cmptType, nComponents) since this method
|
|
||||||
// needs to compile (and work) even with things like strings etc.
|
|
||||||
|
|
||||||
// Single char ad hoc "enum":
|
|
||||||
// - b(yte): gather bytes
|
|
||||||
// - f(loat): gather scalars components
|
|
||||||
// - i(nt): gather label components
|
|
||||||
// - 0: gather with Pstream read/write etc.
|
|
||||||
|
|
||||||
List<int> recvCounts;
|
|
||||||
List<int> recvOffsets;
|
|
||||||
|
|
||||||
char dataMode(0);
|
|
||||||
int nCmpts(0);
|
|
||||||
|
|
||||||
if constexpr (is_contiguous_v<Type>)
|
|
||||||
{
|
{
|
||||||
if constexpr (is_contiguous_scalar<Type>::value)
|
allData.resize_nocopy(offsets_.back()); // == totalSize()
|
||||||
{
|
}
|
||||||
dataMode = 'f';
|
else
|
||||||
nCmpts = static_cast<int>(sizeof(Type)/sizeof(scalar));
|
{
|
||||||
}
|
allData.clear(); // zero-size on non-master
|
||||||
else if constexpr (is_contiguous_label<Type>::value)
|
}
|
||||||
{
|
|
||||||
dataMode = 'i';
|
if constexpr (UPstream_dataType<Type>::value)
|
||||||
nCmpts = static_cast<int>(sizeof(Type)/sizeof(label));
|
{
|
||||||
}
|
// Restrict to basic (or aliased) MPI types
|
||||||
else
|
// - simplifies calculating counts/offsets
|
||||||
{
|
// and can call UPstream::mpiGatherv directly
|
||||||
dataMode = 'b';
|
|
||||||
nCmpts = static_cast<int>(sizeof(Type));
|
// The sizing factor is constexpr
|
||||||
}
|
constexpr std::streamsize count = UPstream_dataType<Type>::size(1);
|
||||||
|
|
||||||
|
static_assert
|
||||||
|
(
|
||||||
|
(count == 1),
|
||||||
|
"Code does not (yet) work with aggregate types"
|
||||||
|
);
|
||||||
|
|
||||||
|
List<int> recvCounts;
|
||||||
|
List<int> recvOffsets;
|
||||||
|
|
||||||
// Offsets must fit into int
|
|
||||||
if (UPstream::master(comm))
|
if (UPstream::master(comm))
|
||||||
{
|
{
|
||||||
const globalIndex& globalAddr = *this;
|
// Must be same as Pstream::nProcs(comm), at least on master!
|
||||||
|
// if (UPstream::nProcs(comm) != this->nProcs()) ...
|
||||||
|
|
||||||
if (globalAddr.totalSize() > (INT_MAX/nCmpts))
|
recvCounts.resize(offsets_.size()-1);
|
||||||
{
|
recvOffsets.resize(offsets_.size());
|
||||||
// Offsets do not fit into int - revert to manual.
|
|
||||||
dataMode = 0;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
// Must be same as Pstream::nProcs(comm), at least on master!
|
|
||||||
const label nproc = globalAddr.nProcs();
|
|
||||||
|
|
||||||
allData.resize_nocopy(globalAddr.totalSize());
|
// Copy offsets
|
||||||
|
std::copy(offsets_.begin(), offsets_.end(), recvOffsets.begin());
|
||||||
|
|
||||||
recvCounts.resize(nproc);
|
// Calculate sizes. Currently without std::minus <functional>
|
||||||
recvOffsets.resize(nproc+1);
|
std::transform
|
||||||
|
(
|
||||||
|
offsets_.begin()+1, offsets_.end(),
|
||||||
|
offsets_.begin(), recvCounts.begin(),
|
||||||
|
std::minus<>{}
|
||||||
|
);
|
||||||
|
|
||||||
for (label proci = 0; proci < nproc; ++proci)
|
// FUTURE .. fix sizes and offsets by the element factor...
|
||||||
{
|
// if constexpr (UPstream_basic_dataType<Type>::size(1) > 1)
|
||||||
recvCounts[proci] = globalAddr.localSize(proci)*nCmpts;
|
|
||||||
recvOffsets[proci] = globalAddr.localStart(proci)*nCmpts;
|
|
||||||
}
|
|
||||||
recvOffsets[nproc] = globalAddr.totalSize()*nCmpts;
|
|
||||||
|
|
||||||
// Assign local data directly
|
|
||||||
|
|
||||||
recvCounts[0] = 0; // ie, ignore for MPI_Gatherv
|
|
||||||
SubList<Type>(allData, globalAddr.range(0)) =
|
|
||||||
SubList<Type>(sendData, globalAddr.range(0));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Consistent information for everyone
|
int sendSize = static_cast<int>(sendData.size());
|
||||||
UPstream::broadcast(&dataMode, 1, comm);
|
|
||||||
|
// Note we let MPI_Gatherv copy back the local data as well...
|
||||||
|
|
||||||
|
UPstream::mpiGatherv
|
||||||
|
(
|
||||||
|
sendData.cdata(),
|
||||||
|
sendSize,
|
||||||
|
allData.data(),
|
||||||
|
recvCounts,
|
||||||
|
recvOffsets,
|
||||||
|
comm
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
else
|
||||||
// Dispatch
|
|
||||||
switch (dataMode)
|
|
||||||
{
|
{
|
||||||
case 'b': // Byte-wise
|
// Regular (manual) gathering
|
||||||
{
|
globalIndex::gather
|
||||||
UPstream::mpiGatherv
|
(
|
||||||
(
|
offsets_, // needed on master only
|
||||||
sendData.cdata_bytes(),
|
comm,
|
||||||
sendData.size_bytes(),
|
UPstream::allProcs(comm), // All communicator ranks
|
||||||
allData.data_bytes(),
|
sendData,
|
||||||
recvCounts,
|
allData,
|
||||||
recvOffsets,
|
tag,
|
||||||
comm
|
commsType
|
||||||
);
|
);
|
||||||
break;
|
|
||||||
}
|
|
||||||
case 'f': // Float (scalar) components
|
|
||||||
{
|
|
||||||
typedef scalar cmptType;
|
|
||||||
|
|
||||||
UPstream::mpiGatherv
|
|
||||||
(
|
|
||||||
reinterpret_cast<const cmptType*>(sendData.cdata()),
|
|
||||||
(sendData.size()*nCmpts),
|
|
||||||
reinterpret_cast<cmptType*>(allData.data()),
|
|
||||||
recvCounts,
|
|
||||||
recvOffsets,
|
|
||||||
comm
|
|
||||||
);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
case 'i': // Int (label) components
|
|
||||||
{
|
|
||||||
typedef label cmptType;
|
|
||||||
|
|
||||||
UPstream::mpiGatherv
|
|
||||||
(
|
|
||||||
reinterpret_cast<const cmptType*>(sendData.cdata()),
|
|
||||||
(sendData.size()*nCmpts),
|
|
||||||
reinterpret_cast<cmptType*>(allData.data()),
|
|
||||||
recvCounts,
|
|
||||||
recvOffsets,
|
|
||||||
comm
|
|
||||||
);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
default: // Regular (manual) gathering
|
|
||||||
{
|
|
||||||
globalIndex::gather
|
|
||||||
(
|
|
||||||
offsets_, // needed on master only
|
|
||||||
comm,
|
|
||||||
UPstream::allProcs(comm), // All communicator ranks
|
|
||||||
sendData,
|
|
||||||
allData,
|
|
||||||
tag,
|
|
||||||
commsType
|
|
||||||
);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!UPstream::master(comm))
|
|
||||||
{
|
|
||||||
allData.clear(); // safety: zero-size on non-master
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -28,33 +28,45 @@ License
|
|||||||
#include "Pstream.H"
|
#include "Pstream.H"
|
||||||
#include "PstreamGlobals.H"
|
#include "PstreamGlobals.H"
|
||||||
#include "UPstreamWrapping.H"
|
#include "UPstreamWrapping.H"
|
||||||
|
#include "vector.H" // for debugging
|
||||||
|
|
||||||
|
#undef STRINGIFY
|
||||||
|
#undef STRING_QUOTE
|
||||||
|
|
||||||
|
#define STRINGIFY(content) #content
|
||||||
|
#define STRING_QUOTE(input) STRINGIFY(input)
|
||||||
|
|
||||||
// * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
|
// * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
|
||||||
|
|
||||||
static inline bool is_basic_dataType(Foam::UPstream::dataTypes id) noexcept
|
namespace
|
||||||
|
{
|
||||||
|
|
||||||
|
inline bool is_nonAggregate(Foam::UPstream::dataTypes id) noexcept
|
||||||
{
|
{
|
||||||
return
|
return
|
||||||
(
|
(
|
||||||
int(id) >= int(Foam::UPstream::dataTypes::Basic_begin)
|
int(id) >= int(Foam::UPstream::dataTypes::Basic_begin)
|
||||||
&& int(id) < int(Foam::UPstream::dataTypes::Basic_end)
|
&& int(id) < int(Foam::UPstream::dataTypes::Basic_end)
|
||||||
|
)
|
||||||
|
||
|
||||||
|
(
|
||||||
|
int(id) >= int(Foam::UPstream::dataTypes::User_begin)
|
||||||
|
&& int(id) < int(Foam::UPstream::dataTypes::User_end)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
namespace
|
|
||||||
{
|
|
||||||
|
|
||||||
using namespace Foam;
|
|
||||||
|
|
||||||
// Local function to print some error information
|
// Local function to print some error information
|
||||||
inline void printErrorNonIntrinsic
|
inline void printErrorNonIntrinsic
|
||||||
(
|
(
|
||||||
const char* context,
|
const char* context,
|
||||||
UPstream::dataTypes dataTypeId
|
Foam::UPstream::dataTypes dataTypeId
|
||||||
)
|
)
|
||||||
{
|
{
|
||||||
|
using namespace Foam;
|
||||||
|
|
||||||
FatalError
|
FatalError
|
||||||
<< "Bad input for " << context << ": likely a programming problem\n"
|
<< "Bad input for " << context << ": likely a programming problem\n"
|
||||||
<< " Non-intrinsic data (" << int(dataTypeId) << ")\n"
|
<< " Non-intrinsic/non-user data (type:" << int(dataTypeId) << ")\n"
|
||||||
<< Foam::endl;
|
<< Foam::endl;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -214,19 +226,37 @@ void Foam::UPstream::mpi_gatherv
|
|||||||
{
|
{
|
||||||
MPI_Datatype datatype = PstreamGlobals::getDataType(dataTypeId);
|
MPI_Datatype datatype = PstreamGlobals::getDataType(dataTypeId);
|
||||||
|
|
||||||
if
|
// Runtime assert that we are not using aggregated data types
|
||||||
(
|
if (FOAM_UNLIKELY(!is_nonAggregate(dataTypeId)))
|
||||||
FOAM_UNLIKELY
|
|
||||||
(
|
|
||||||
!is_basic_dataType(dataTypeId)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
{
|
{
|
||||||
FatalErrorInFunction;
|
FatalErrorInFunction;
|
||||||
printErrorNonIntrinsic("MPI_Gatherv()", dataTypeId);
|
printErrorNonIntrinsic("MPI_Gatherv()", dataTypeId);
|
||||||
FatalError << Foam::abort(FatalError);
|
FatalError << Foam::abort(FatalError);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const label np = UPstream::nProcs(communicator);
|
||||||
|
|
||||||
|
// For total-size calculation,
|
||||||
|
// don't rely on recvOffsets being (np+1)
|
||||||
|
const int totalSize =
|
||||||
|
(
|
||||||
|
(UPstream::master(communicator) && np > 1)
|
||||||
|
? (recvOffsets[np-1] + recvCounts[np-1])
|
||||||
|
: 0
|
||||||
|
);
|
||||||
|
|
||||||
|
if (FOAM_UNLIKELY(UPstream::debug))
|
||||||
|
{
|
||||||
|
Perr<< "[mpi_gatherv] :"
|
||||||
|
<< " type:" << int(dataTypeId)
|
||||||
|
<< " count:" << sendCount
|
||||||
|
<< " total:" << totalSize
|
||||||
|
<< " comm:" << communicator
|
||||||
|
<< " recvCounts:" << flatOutput(recvCounts)
|
||||||
|
<< " recvOffsets:" << flatOutput(recvOffsets)
|
||||||
|
<< Foam::endl;
|
||||||
|
}
|
||||||
|
|
||||||
{
|
{
|
||||||
PstreamDetail::gatherv
|
PstreamDetail::gatherv
|
||||||
(
|
(
|
||||||
@ -235,6 +265,48 @@ void Foam::UPstream::mpi_gatherv
|
|||||||
datatype, communicator
|
datatype, communicator
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Extended debugging. Limit to master:
|
||||||
|
|
||||||
|
#if 0
|
||||||
|
if (FOAM_UNLIKELY(UPstream::debug))
|
||||||
|
{
|
||||||
|
if (UPstream::master(communicator))
|
||||||
|
{
|
||||||
|
switch (dataTypeId)
|
||||||
|
{
|
||||||
|
#undef dataPrinter
|
||||||
|
#define dataPrinter(enumType, nativeType) \
|
||||||
|
case UPstream::dataTypes::enumType : \
|
||||||
|
{ \
|
||||||
|
UList<nativeType> combined \
|
||||||
|
( \
|
||||||
|
static_cast<nativeType*>(recvData), \
|
||||||
|
totalSize \
|
||||||
|
); \
|
||||||
|
\
|
||||||
|
Info<< "[mpi_gatherv] => " \
|
||||||
|
"List<" STRING_QUOTE(nativeType) "> "; \
|
||||||
|
combined.writeList(Info) << Foam::endl; \
|
||||||
|
\
|
||||||
|
break; \
|
||||||
|
}
|
||||||
|
|
||||||
|
// Some common types
|
||||||
|
dataPrinter(type_int32, int32_t);
|
||||||
|
dataPrinter(type_int64, int64_t);
|
||||||
|
dataPrinter(type_float, float);
|
||||||
|
dataPrinter(type_double, double);
|
||||||
|
dataPrinter(type_3float, floatVector);
|
||||||
|
dataPrinter(type_3double, doubleVector);
|
||||||
|
|
||||||
|
// Some other type
|
||||||
|
default: break;
|
||||||
|
#undef dataPrinter
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -255,13 +327,8 @@ void Foam::UPstream::mpi_scatterv
|
|||||||
{
|
{
|
||||||
MPI_Datatype datatype = PstreamGlobals::getDataType(dataTypeId);
|
MPI_Datatype datatype = PstreamGlobals::getDataType(dataTypeId);
|
||||||
|
|
||||||
if
|
// Runtime assert that we are not using aggregated data types
|
||||||
(
|
if (FOAM_UNLIKELY(!is_nonAggregate(dataTypeId)))
|
||||||
FOAM_UNLIKELY
|
|
||||||
(
|
|
||||||
!is_basic_dataType(dataTypeId)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
{
|
{
|
||||||
FatalErrorInFunction;
|
FatalErrorInFunction;
|
||||||
printErrorNonIntrinsic("MPI_Scatterv()", dataTypeId);
|
printErrorNonIntrinsic("MPI_Scatterv()", dataTypeId);
|
||||||
|
|||||||
@ -201,6 +201,7 @@ Foam::surfaceWriter::surfaceWriter()
|
|||||||
isPointData_(false),
|
isPointData_(false),
|
||||||
verbose_(false),
|
verbose_(false),
|
||||||
commType_(UPstream::commsTypes::scheduled),
|
commType_(UPstream::commsTypes::scheduled),
|
||||||
|
gatherv_(false),
|
||||||
nFields_(0),
|
nFields_(0),
|
||||||
currTime_(),
|
currTime_(),
|
||||||
outputPath_(),
|
outputPath_(),
|
||||||
@ -218,6 +219,8 @@ Foam::surfaceWriter::surfaceWriter(const dictionary& options)
|
|||||||
options.readIfPresent("verbose", verbose_);
|
options.readIfPresent("verbose", verbose_);
|
||||||
|
|
||||||
UPstream::commsTypeNames.readIfPresent("commsType", options, commType_);
|
UPstream::commsTypeNames.readIfPresent("commsType", options, commType_);
|
||||||
|
gatherv_ = false;
|
||||||
|
options.readIfPresent("gatherv", gatherv_);
|
||||||
|
|
||||||
geometryScale_ = 1;
|
geometryScale_ = 1;
|
||||||
geometryCentre_ = Zero;
|
geometryCentre_ = Zero;
|
||||||
@ -244,7 +247,19 @@ Foam::surfaceWriter::surfaceWriter(const dictionary& options)
|
|||||||
{
|
{
|
||||||
Info<< "Create surfaceWriter ("
|
Info<< "Create surfaceWriter ("
|
||||||
<< (this->isPointData() ? "point" : "face") << " data):"
|
<< (this->isPointData() ? "point" : "face") << " data):"
|
||||||
<< " commsType=" << UPstream::commsTypeNames[commType_] << endl;
|
<< " commsType=";
|
||||||
|
|
||||||
|
if (UPstream::parRun())
|
||||||
|
{
|
||||||
|
if (gatherv_) Info<< "gatherv+";
|
||||||
|
Info<< UPstream::commsTypeNames[commType_];
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
Info<< "serial";
|
||||||
|
}
|
||||||
|
|
||||||
|
Info<< endl;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -605,14 +620,29 @@ Foam::tmp<Foam::Field<Type>> Foam::surfaceWriter::mergeFieldTemplate
|
|||||||
: mergedSurf_.faceGlobalIndex()
|
: mergedSurf_.faceGlobalIndex()
|
||||||
);
|
);
|
||||||
|
|
||||||
globIndex.gather
|
if (gatherv_)
|
||||||
(
|
{
|
||||||
fld,
|
globIndex.mpiGather
|
||||||
allFld,
|
(
|
||||||
UPstream::msgType(),
|
fld,
|
||||||
commType_,
|
allFld,
|
||||||
UPstream::worldComm
|
UPstream::worldComm,
|
||||||
);
|
// For fallback:
|
||||||
|
commType_,
|
||||||
|
UPstream::msgType()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
globIndex.gather
|
||||||
|
(
|
||||||
|
fld,
|
||||||
|
allFld,
|
||||||
|
UPstream::msgType(),
|
||||||
|
commType_,
|
||||||
|
UPstream::worldComm
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
// Renumber (point data) to correspond to merged points
|
// Renumber (point data) to correspond to merged points
|
||||||
if
|
if
|
||||||
@ -626,6 +656,15 @@ Foam::tmp<Foam::Field<Type>> Foam::surfaceWriter::mergeFieldTemplate
|
|||||||
allFld.resize(mergedSurf_.points().size());
|
allFld.resize(mergedSurf_.points().size());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Extended debugging. Limit to master:
|
||||||
|
#if 0
|
||||||
|
if (UPstream::master())
|
||||||
|
{
|
||||||
|
Info<< "merged List<" << pTraits<Type>::typeName << "> : ";
|
||||||
|
allFld.writeList(Info) << endl;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
return tfield;
|
return tfield;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -6,7 +6,7 @@
|
|||||||
\\/ M anipulation |
|
\\/ M anipulation |
|
||||||
-------------------------------------------------------------------------------
|
-------------------------------------------------------------------------------
|
||||||
Copyright (C) 2011-2012 OpenFOAM Foundation
|
Copyright (C) 2011-2012 OpenFOAM Foundation
|
||||||
Copyright (C) 2015-2024 OpenCFD Ltd.
|
Copyright (C) 2015-2025 OpenCFD Ltd.
|
||||||
-------------------------------------------------------------------------------
|
-------------------------------------------------------------------------------
|
||||||
License
|
License
|
||||||
This file is part of OpenFOAM.
|
This file is part of OpenFOAM.
|
||||||
@ -82,6 +82,7 @@ Description
|
|||||||
Property | Description | Reqd | Default
|
Property | Description | Reqd | Default
|
||||||
verbose | Additional output verbosity | no | no
|
verbose | Additional output verbosity | no | no
|
||||||
commsType | Communication type | no | scheduled
|
commsType | Communication type | no | scheduled
|
||||||
|
gatherv | Use MPI gatherv [experimental] | no | false
|
||||||
scale | Output geometry scaling | no | 1
|
scale | Output geometry scaling | no | 1
|
||||||
transform | Output coordinate transform | no |
|
transform | Output coordinate transform | no |
|
||||||
fieldLevel | Subtract field level before scaling | no | empty dict
|
fieldLevel | Subtract field level before scaling | no | empty dict
|
||||||
@ -97,6 +98,9 @@ Note
|
|||||||
it is the responsibility of the implementation (not the caller)
|
it is the responsibility of the implementation (not the caller)
|
||||||
to ensure that this occurs.
|
to ensure that this occurs.
|
||||||
|
|
||||||
|
Using MPI gatherv [experimental] is not well tested and may change
|
||||||
|
or be removed in the future!
|
||||||
|
|
||||||
SourceFiles
|
SourceFiles
|
||||||
surfaceWriter.C
|
surfaceWriter.C
|
||||||
surfaceWriterI.H
|
surfaceWriterI.H
|
||||||
@ -191,6 +195,9 @@ protected:
|
|||||||
//- Communication type (for field merging)
|
//- Communication type (for field merging)
|
||||||
UPstream::commsTypes commType_;
|
UPstream::commsTypes commType_;
|
||||||
|
|
||||||
|
//- Prefer MPI gatherv intrinsic (for field merging) [experimental]
|
||||||
|
bool gatherv_;
|
||||||
|
|
||||||
//- The number of fields
|
//- The number of fields
|
||||||
label nFields_;
|
label nFields_;
|
||||||
|
|
||||||
|
|||||||
@ -88,8 +88,11 @@ Foam::surfaceWriters::debugWriter::debugWriter
|
|||||||
streamOpt_(IOstreamOption::BINARY)
|
streamOpt_(IOstreamOption::BINARY)
|
||||||
{
|
{
|
||||||
Info<< "Using debug surface writer ("
|
Info<< "Using debug surface writer ("
|
||||||
<< (this->isPointData() ? "point" : "face") << " data):"
|
<< (this->isPointData() ? "point" : "face") << " data):";
|
||||||
<< " commsType=" << UPstream::commsTypeNames[commType_]
|
|
||||||
|
if (gatherv_) Info<< " <gatherv>";
|
||||||
|
|
||||||
|
Info<< " commsType=" << UPstream::commsTypeNames[commType_]
|
||||||
<< " merge=" << Switch::name(enableMerge_)
|
<< " merge=" << Switch::name(enableMerge_)
|
||||||
<< " write=" << Switch::name(enableWrite_) << endl;
|
<< " write=" << Switch::name(enableWrite_) << endl;
|
||||||
}
|
}
|
||||||
|
|||||||
11
tutorials/incompressible/simpleFoam/squareBend/Allclean
Executable file
11
tutorials/incompressible/simpleFoam/squareBend/Allclean
Executable file
@ -0,0 +1,11 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
cd "${0%/*}" || exit # Run from this directory
|
||||||
|
. ${WM_PROJECT_DIR:?}/bin/tools/CleanFunctions # Tutorial clean functions
|
||||||
|
#------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
cleanCase0
|
||||||
|
|
||||||
|
# Remove surface and features
|
||||||
|
rm -rf constant/triSurface
|
||||||
|
|
||||||
|
#------------------------------------------------------------------------------
|
||||||
@ -49,7 +49,8 @@ __surfaceFieldValue
|
|||||||
{
|
{
|
||||||
default
|
default
|
||||||
{
|
{
|
||||||
verbose true;
|
verbose true;
|
||||||
|
//gatherv true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -19,6 +19,12 @@ debug
|
|||||||
|
|
||||||
formatOptions
|
formatOptions
|
||||||
{
|
{
|
||||||
|
default
|
||||||
|
{
|
||||||
|
verbose true;
|
||||||
|
//gatherv true;
|
||||||
|
}
|
||||||
|
|
||||||
ensight
|
ensight
|
||||||
{
|
{
|
||||||
collateTimes true;
|
collateTimes true;
|
||||||
|
|||||||
Reference in New Issue
Block a user