ENH: update globalIndex::mpiGather to use MPI intrinsic/user types

- add provisional support for selecting MPI_Gatherv
  when merging fields in the surface writers.
  Uses the 'gatherv' keyword [experimental]

BUG: gatherv/scatterv wrappers used the incorrect data type

- incorrectly checked against UPstream_basic_dataType trait instead of
  UPstream_dataType, which resulted in a count mismatch for
  user-defined types (eg, vector, tensor,...).

  This was not visibly active bug, since previous internal uses of
  gatherv were restricted to primitive types.

COMP: make UPstream dataType traits size(...) constexpr

- allows static asserts and/or compile-time selection
  of code based on size(1)
This commit is contained in:
Mark Olesen
2025-10-09 12:03:33 +02:00
parent 55c81bce1b
commit 4b92bb6533
14 changed files with 351 additions and 202 deletions

View File

@ -200,7 +200,7 @@ void printTypeName()
template<class Type, bool UseTypeName = true> template<class Type, bool UseTypeName = true>
void printPstreamTraits(const std::string_view name = std::string_view()) void printPstreamTraits(std::string_view name = std::string_view())
{ {
Info<< "========" << nl; Info<< "========" << nl;
Info<< "type: "; Info<< "type: ";
@ -299,6 +299,9 @@ void printPstreamTraits(const std::string_view name = std::string_view())
// Use element or component type (or byte-wise) for data type // Use element or component type (or byte-wise) for data type
using base = typename UPstream_dataType<Type>::base; using base = typename UPstream_dataType<Type>::base;
// The sizing factor is constexpr
constexpr std::streamsize count = UPstream_dataType<Type>::size(1);
Info<< " : "; Info<< " : ";
if constexpr (UseTypeName) if constexpr (UseTypeName)
{ {
@ -311,8 +314,7 @@ void printPstreamTraits(const std::string_view name = std::string_view())
Info<< " cmpt-type="; Info<< " cmpt-type=";
printDataTypeId(UPstream_dataType<Type>::datatype_id); printDataTypeId(UPstream_dataType<Type>::datatype_id);
Info<< " count=" << UPstream_dataType<Type>::size(1); Info<< " count=" << count << nl;
Info<< nl;
} }
} }
@ -362,6 +364,24 @@ void print_data_opType(BinaryOp bop, std::string_view name)
} }
template<class Type>
int check_simple(std::string_view name = std::string_view())
{
// The sizing factor is constexpr
constexpr std::streamsize count = UPstream_dataType<Type>::size(1);
static_assert
(
(count == 1),
"Code does not (yet) work with aggregate types"
);
Info<< "check_simple: " << name << ": " << count << nl;
return count;
}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
// Main program: // Main program:
@ -389,6 +409,8 @@ int main()
printPstreamTraits<const float>(); printPstreamTraits<const float>();
printPstreamTraits<floatVector>(); printPstreamTraits<floatVector>();
check_simple<floatVector>("vector<float>");
printPstreamTraits<scalar>(); printPstreamTraits<scalar>();
printPstreamTraits<double>(); printPstreamTraits<double>();
printPstreamTraits<doubleVector>(); printPstreamTraits<doubleVector>();

View File

@ -1,4 +1,4 @@
mydebugSurfaceWriter.C mydebugSurfaceWriter.cxx
Test-surface-sampling.C Test-surface-sampling.cxx
EXE = $(FOAM_USER_APPBIN)/Test-surface-sampling EXE = $(FOAM_USER_APPBIN)/Test-surface-sampling

View File

@ -5,7 +5,7 @@
\\ / A nd | www.openfoam.com \\ / A nd | www.openfoam.com
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2022-2023 OpenCFD Ltd. Copyright (C) 2022-2025 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
@ -83,12 +83,10 @@ template<> struct narrowType<SymmTensor<double>>
typedef SymmTensor<float> type; typedef SymmTensor<float> type;
}; };
// FIXME: Not sure why this one seems to be broken... template<> struct narrowType<Tensor<double>>
// {
// template<> struct narrowType<Tensor<double>> typedef Tensor<float> type;
// { };
// typedef Tensor<float> type;
// };
} // End namespace Foam } // End namespace Foam
@ -104,12 +102,18 @@ Foam::surfaceWriters::mydebugWriter::mergeField
{ {
addProfiling(merge, "debugWriter::merge-field"); addProfiling(merge, "debugWriter::merge-field");
// This is largely identical to surfaceWriter::mergeField() // Identical to surfaceWriter::mergeField()
// but with narrowing for communication // but with narrowing for communication
if (narrowTransfer_ && parallel_ && UPstream::parRun())
if constexpr (std::is_same_v<Tensor<double>, Type>)
{
// Cannot narrow tensor. Does not compile since MatrixSpace
// does not (yet) allow assigments from different Cmpt types.
}
else if (narrowTransfer_ && parallel_ && UPstream::parRun())
{ {
// The narrowed type // The narrowed type
typedef typename narrowType<Type>::type narrowedType; using narrowedType = typename narrowType<Type>::type;
// Ensure geometry is also merged // Ensure geometry is also merged
merge(); merge();
@ -130,14 +134,29 @@ Foam::surfaceWriters::mydebugWriter::mergeField
ConstPrecisionAdaptor<narrowedType, Type> input(fld); ConstPrecisionAdaptor<narrowedType, Type> input(fld);
PrecisionAdaptor<narrowedType, Type> output(allFld); PrecisionAdaptor<narrowedType, Type> output(allFld);
globIndex.gather if (gatherv_)
( {
input.cref(), // fld, globIndex.mpiGather
output.ref(), // allFld, (
UPstream::msgType(), input.cref(), // fld
commType_, output.ref(), // allFld
UPstream::worldComm UPstream::worldComm,
); // For fallback:
commType_,
UPstream::msgType()
);
}
else
{
globIndex.gather
(
input.cref(), // fld
output.ref(), // allFld
UPstream::msgType(),
commType_,
UPstream::worldComm
);
}
// Commit adapted content changes // Commit adapted content changes
input.commit(); input.commit();
@ -193,8 +212,19 @@ Foam::surfaceWriters::mydebugWriter::mydebugWriter
{ {
Info<< "Using debug surface writer (" Info<< "Using debug surface writer ("
<< (this->isPointData() ? "point" : "face") << " data):" << (this->isPointData() ? "point" : "face") << " data):"
<< " commsType=" << UPstream::commsTypeNames[commType_] << " commsType=";
<< " merge=" << Switch::name(enableMerge_)
if (UPstream::parRun())
{
if (gatherv_) Info<< "gatherv+";
Info<< UPstream::commsTypeNames[commType_];
}
else
{
Info<< "serial";
}
Info<< " merge=" << Switch::name(enableMerge_)
<< " write=" << Switch::name(enableWrite_) << " write=" << Switch::name(enableWrite_)
<< " narrow=" << Switch::name(narrowTransfer_) << " narrow=" << Switch::name(narrowTransfer_)
<< endl; << endl;

View File

@ -343,11 +343,22 @@ void Foam::UPstream::mpiGatherv
} }
// Nothing further to do // Nothing further to do
} }
else if constexpr (UPstream_basic_dataType<Type>::value) else if constexpr (UPstream_dataType<Type>::value)
{ {
// Restrict to basic (or aliased) MPI types to avoid recalculating // Restrict to basic (or aliased) MPI types to avoid recalculating
// the list of counts/offsets. // the list of counts/offsets.
// The sizing factor (constexpr) must be 1 otherwise
// [recvCounts,recvOffsets] are likely incorrect
constexpr std::streamsize count = UPstream_dataType<Type>::size(1);
static_assert
(
(count == 1),
"Code does not (yet) work with aggregate types"
);
UPstream::mpi_gatherv UPstream::mpi_gatherv
( (
sendData, sendData,
@ -356,7 +367,7 @@ void Foam::UPstream::mpiGatherv
recvCounts, recvCounts,
recvOffsets, recvOffsets,
UPstream_basic_dataType<Type>::datatype_id, UPstream_dataType<Type>::datatype_id,
communicator communicator
); );
} }
@ -364,7 +375,8 @@ void Foam::UPstream::mpiGatherv
{ {
static_assert static_assert
( (
stdFoam::dependent_false_v<Type>, "Only basic MPI data types" stdFoam::dependent_false_v<Type>,
"Only basic and user data types"
); );
} }
} }
@ -392,11 +404,22 @@ void Foam::UPstream::mpiScatterv
} }
// Nothing further to do // Nothing further to do
} }
else if constexpr (UPstream_basic_dataType<Type>::value) else if constexpr (UPstream_dataType<Type>::value)
{ {
// Restrict to basic (or aliased) MPI types to avoid recalculating // Restrict to basic (or aliased) MPI types to avoid recalculating
// the list of counts/offsets. // the list of counts/offsets.
// The sizing factor (constexpr) must be 1 otherwise
// [sendCounts,sendOffsets] are likely incorrect
constexpr std::streamsize count = UPstream_dataType<Type>::size(1);
static_assert
(
(count == 1),
"Code does not (yet) work with aggregate types"
);
UPstream::mpi_scatterv UPstream::mpi_scatterv
( (
sendData, sendData,
@ -405,7 +428,7 @@ void Foam::UPstream::mpiScatterv
recvData, recvData,
recvCount, recvCount,
UPstream_basic_dataType<Type>::datatype_id, UPstream_dataType<Type>::datatype_id,
communicator communicator
); );
} }
@ -413,7 +436,8 @@ void Foam::UPstream::mpiScatterv
{ {
static_assert static_assert
( (
stdFoam::dependent_false_v<Type>, "Only basic MPI data types" stdFoam::dependent_false_v<Type>,
"Only basic and user data types"
); );
} }
} }

View File

@ -354,7 +354,7 @@ struct UPstream_basic_dataType
UPstream_alias_dataType<base>::datatype_id; UPstream_alias_dataType<base>::datatype_id;
//- The size in terms of the number of underlying data elements //- The size in terms of the number of underlying data elements
static std::streamsize size(std::streamsize n) noexcept static constexpr std::streamsize size(std::streamsize n) noexcept
{ {
if constexpr (UPstream_alias_dataType<T>::value) if constexpr (UPstream_alias_dataType<T>::value)
{ {
@ -373,7 +373,10 @@ struct UPstream_basic_dataType
template<> struct UPstream_basic_dataType<void> : UPstream_mpi_dataType<void> template<> struct UPstream_basic_dataType<void> : UPstream_mpi_dataType<void>
{ {
using base = void; using base = void;
static std::streamsize size(std::streamsize n) noexcept { return n; } static constexpr std::streamsize size(std::streamsize n) noexcept
{
return n;
}
}; };
@ -410,7 +413,7 @@ struct UPstream_dataType
UPstream_any_dataType<base>::datatype_id; UPstream_any_dataType<base>::datatype_id;
//- The size in terms of the number of base data elements //- The size in terms of the number of base data elements
static std::streamsize size(std::streamsize n) noexcept static constexpr std::streamsize size(std::streamsize n) noexcept
{ {
if constexpr (UPstream_any_dataType<T>::value) if constexpr (UPstream_any_dataType<T>::value)
{ {

View File

@ -27,6 +27,7 @@ License
\*---------------------------------------------------------------------------*/ \*---------------------------------------------------------------------------*/
#include "globalIndex.H" #include "globalIndex.H"
#include <functional>
// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * // // * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * //
@ -696,159 +697,94 @@ void Foam::globalIndex::mpiGather
const UList<Type>& sendData, const UList<Type>& sendData,
OutputContainer& allData, OutputContainer& allData,
const label comm, const label comm,
UPstream::commsTypes commsType, [[maybe_unused]] UPstream::commsTypes commsType,
const int tag [[maybe_unused]] const int tag
) const ) const
{ {
if (!UPstream::parRun()) if (!UPstream::is_parallel(comm))
{ {
// Serial: direct copy // Serial: direct copy
allData = sendData; allData = sendData;
return; return;
} }
// MPI_Gatherv requires contiguous data, but a byte-wise transfer can if (UPstream::master(comm))
// quickly exceed the 'int' limits used for MPI sizes/offsets.
// Thus gather label/scalar components when possible to increase the
// effective size limit.
//
// Note: cannot rely on pTraits (cmptType, nComponents) since this method
// needs to compile (and work) even with things like strings etc.
// Single char ad hoc "enum":
// - b(yte): gather bytes
// - f(loat): gather scalars components
// - i(nt): gather label components
// - 0: gather with Pstream read/write etc.
List<int> recvCounts;
List<int> recvOffsets;
char dataMode(0);
int nCmpts(0);
if constexpr (is_contiguous_v<Type>)
{ {
if constexpr (is_contiguous_scalar<Type>::value) allData.resize_nocopy(offsets_.back()); // == totalSize()
{ }
dataMode = 'f'; else
nCmpts = static_cast<int>(sizeof(Type)/sizeof(scalar)); {
} allData.clear(); // zero-size on non-master
else if constexpr (is_contiguous_label<Type>::value) }
{
dataMode = 'i'; if constexpr (UPstream_dataType<Type>::value)
nCmpts = static_cast<int>(sizeof(Type)/sizeof(label)); {
} // Restrict to basic (or aliased) MPI types
else // - simplifies calculating counts/offsets
{ // and can call UPstream::mpiGatherv directly
dataMode = 'b';
nCmpts = static_cast<int>(sizeof(Type)); // The sizing factor is constexpr
} constexpr std::streamsize count = UPstream_dataType<Type>::size(1);
static_assert
(
(count == 1),
"Code does not (yet) work with aggregate types"
);
List<int> recvCounts;
List<int> recvOffsets;
// Offsets must fit into int
if (UPstream::master(comm)) if (UPstream::master(comm))
{ {
const globalIndex& globalAddr = *this; // Must be same as Pstream::nProcs(comm), at least on master!
// if (UPstream::nProcs(comm) != this->nProcs()) ...
if (globalAddr.totalSize() > (INT_MAX/nCmpts)) recvCounts.resize(offsets_.size()-1);
{ recvOffsets.resize(offsets_.size());
// Offsets do not fit into int - revert to manual.
dataMode = 0;
}
else
{
// Must be same as Pstream::nProcs(comm), at least on master!
const label nproc = globalAddr.nProcs();
allData.resize_nocopy(globalAddr.totalSize()); // Copy offsets
std::copy(offsets_.begin(), offsets_.end(), recvOffsets.begin());
recvCounts.resize(nproc); // Calculate sizes. Currently without std::minus <functional>
recvOffsets.resize(nproc+1); std::transform
(
offsets_.begin()+1, offsets_.end(),
offsets_.begin(), recvCounts.begin(),
std::minus<>{}
);
for (label proci = 0; proci < nproc; ++proci) // FUTURE .. fix sizes and offsets by the element factor...
{ // if constexpr (UPstream_basic_dataType<Type>::size(1) > 1)
recvCounts[proci] = globalAddr.localSize(proci)*nCmpts;
recvOffsets[proci] = globalAddr.localStart(proci)*nCmpts;
}
recvOffsets[nproc] = globalAddr.totalSize()*nCmpts;
// Assign local data directly
recvCounts[0] = 0; // ie, ignore for MPI_Gatherv
SubList<Type>(allData, globalAddr.range(0)) =
SubList<Type>(sendData, globalAddr.range(0));
}
} }
// Consistent information for everyone int sendSize = static_cast<int>(sendData.size());
UPstream::broadcast(&dataMode, 1, comm);
// Note we let MPI_Gatherv copy back the local data as well...
UPstream::mpiGatherv
(
sendData.cdata(),
sendSize,
allData.data(),
recvCounts,
recvOffsets,
comm
);
} }
else
// Dispatch
switch (dataMode)
{ {
case 'b': // Byte-wise // Regular (manual) gathering
{ globalIndex::gather
UPstream::mpiGatherv (
( offsets_, // needed on master only
sendData.cdata_bytes(), comm,
sendData.size_bytes(), UPstream::allProcs(comm), // All communicator ranks
allData.data_bytes(), sendData,
recvCounts, allData,
recvOffsets, tag,
comm commsType
); );
break;
}
case 'f': // Float (scalar) components
{
typedef scalar cmptType;
UPstream::mpiGatherv
(
reinterpret_cast<const cmptType*>(sendData.cdata()),
(sendData.size()*nCmpts),
reinterpret_cast<cmptType*>(allData.data()),
recvCounts,
recvOffsets,
comm
);
break;
}
case 'i': // Int (label) components
{
typedef label cmptType;
UPstream::mpiGatherv
(
reinterpret_cast<const cmptType*>(sendData.cdata()),
(sendData.size()*nCmpts),
reinterpret_cast<cmptType*>(allData.data()),
recvCounts,
recvOffsets,
comm
);
break;
}
default: // Regular (manual) gathering
{
globalIndex::gather
(
offsets_, // needed on master only
comm,
UPstream::allProcs(comm), // All communicator ranks
sendData,
allData,
tag,
commsType
);
break;
}
}
if (!UPstream::master(comm))
{
allData.clear(); // safety: zero-size on non-master
} }
} }

View File

@ -28,33 +28,45 @@ License
#include "Pstream.H" #include "Pstream.H"
#include "PstreamGlobals.H" #include "PstreamGlobals.H"
#include "UPstreamWrapping.H" #include "UPstreamWrapping.H"
#include "vector.H" // for debugging
#undef STRINGIFY
#undef STRING_QUOTE
#define STRINGIFY(content) #content
#define STRING_QUOTE(input) STRINGIFY(input)
// * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * // // * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
static inline bool is_basic_dataType(Foam::UPstream::dataTypes id) noexcept namespace
{
inline bool is_nonAggregate(Foam::UPstream::dataTypes id) noexcept
{ {
return return
( (
int(id) >= int(Foam::UPstream::dataTypes::Basic_begin) int(id) >= int(Foam::UPstream::dataTypes::Basic_begin)
&& int(id) < int(Foam::UPstream::dataTypes::Basic_end) && int(id) < int(Foam::UPstream::dataTypes::Basic_end)
)
||
(
int(id) >= int(Foam::UPstream::dataTypes::User_begin)
&& int(id) < int(Foam::UPstream::dataTypes::User_end)
); );
} }
namespace
{
using namespace Foam;
// Local function to print some error information // Local function to print some error information
inline void printErrorNonIntrinsic inline void printErrorNonIntrinsic
( (
const char* context, const char* context,
UPstream::dataTypes dataTypeId Foam::UPstream::dataTypes dataTypeId
) )
{ {
using namespace Foam;
FatalError FatalError
<< "Bad input for " << context << ": likely a programming problem\n" << "Bad input for " << context << ": likely a programming problem\n"
<< " Non-intrinsic data (" << int(dataTypeId) << ")\n" << " Non-intrinsic/non-user data (type:" << int(dataTypeId) << ")\n"
<< Foam::endl; << Foam::endl;
} }
@ -214,19 +226,37 @@ void Foam::UPstream::mpi_gatherv
{ {
MPI_Datatype datatype = PstreamGlobals::getDataType(dataTypeId); MPI_Datatype datatype = PstreamGlobals::getDataType(dataTypeId);
if // Runtime assert that we are not using aggregated data types
( if (FOAM_UNLIKELY(!is_nonAggregate(dataTypeId)))
FOAM_UNLIKELY
(
!is_basic_dataType(dataTypeId)
)
)
{ {
FatalErrorInFunction; FatalErrorInFunction;
printErrorNonIntrinsic("MPI_Gatherv()", dataTypeId); printErrorNonIntrinsic("MPI_Gatherv()", dataTypeId);
FatalError << Foam::abort(FatalError); FatalError << Foam::abort(FatalError);
} }
const label np = UPstream::nProcs(communicator);
// For total-size calculation,
// don't rely on recvOffsets being (np+1)
const int totalSize =
(
(UPstream::master(communicator) && np > 1)
? (recvOffsets[np-1] + recvCounts[np-1])
: 0
);
if (FOAM_UNLIKELY(UPstream::debug))
{
Perr<< "[mpi_gatherv] :"
<< " type:" << int(dataTypeId)
<< " count:" << sendCount
<< " total:" << totalSize
<< " comm:" << communicator
<< " recvCounts:" << flatOutput(recvCounts)
<< " recvOffsets:" << flatOutput(recvOffsets)
<< Foam::endl;
}
{ {
PstreamDetail::gatherv PstreamDetail::gatherv
( (
@ -235,6 +265,48 @@ void Foam::UPstream::mpi_gatherv
datatype, communicator datatype, communicator
); );
} }
// Extended debugging. Limit to master:
#if 0
if (FOAM_UNLIKELY(UPstream::debug))
{
if (UPstream::master(communicator))
{
switch (dataTypeId)
{
#undef dataPrinter
#define dataPrinter(enumType, nativeType) \
case UPstream::dataTypes::enumType : \
{ \
UList<nativeType> combined \
( \
static_cast<nativeType*>(recvData), \
totalSize \
); \
\
Info<< "[mpi_gatherv] => " \
"List<" STRING_QUOTE(nativeType) "> "; \
combined.writeList(Info) << Foam::endl; \
\
break; \
}
// Some common types
dataPrinter(type_int32, int32_t);
dataPrinter(type_int64, int64_t);
dataPrinter(type_float, float);
dataPrinter(type_double, double);
dataPrinter(type_3float, floatVector);
dataPrinter(type_3double, doubleVector);
// Some other type
default: break;
#undef dataPrinter
}
}
}
#endif
} }
@ -255,13 +327,8 @@ void Foam::UPstream::mpi_scatterv
{ {
MPI_Datatype datatype = PstreamGlobals::getDataType(dataTypeId); MPI_Datatype datatype = PstreamGlobals::getDataType(dataTypeId);
if // Runtime assert that we are not using aggregated data types
( if (FOAM_UNLIKELY(!is_nonAggregate(dataTypeId)))
FOAM_UNLIKELY
(
!is_basic_dataType(dataTypeId)
)
)
{ {
FatalErrorInFunction; FatalErrorInFunction;
printErrorNonIntrinsic("MPI_Scatterv()", dataTypeId); printErrorNonIntrinsic("MPI_Scatterv()", dataTypeId);

View File

@ -201,6 +201,7 @@ Foam::surfaceWriter::surfaceWriter()
isPointData_(false), isPointData_(false),
verbose_(false), verbose_(false),
commType_(UPstream::commsTypes::scheduled), commType_(UPstream::commsTypes::scheduled),
gatherv_(false),
nFields_(0), nFields_(0),
currTime_(), currTime_(),
outputPath_(), outputPath_(),
@ -218,6 +219,8 @@ Foam::surfaceWriter::surfaceWriter(const dictionary& options)
options.readIfPresent("verbose", verbose_); options.readIfPresent("verbose", verbose_);
UPstream::commsTypeNames.readIfPresent("commsType", options, commType_); UPstream::commsTypeNames.readIfPresent("commsType", options, commType_);
gatherv_ = false;
options.readIfPresent("gatherv", gatherv_);
geometryScale_ = 1; geometryScale_ = 1;
geometryCentre_ = Zero; geometryCentre_ = Zero;
@ -244,7 +247,19 @@ Foam::surfaceWriter::surfaceWriter(const dictionary& options)
{ {
Info<< "Create surfaceWriter (" Info<< "Create surfaceWriter ("
<< (this->isPointData() ? "point" : "face") << " data):" << (this->isPointData() ? "point" : "face") << " data):"
<< " commsType=" << UPstream::commsTypeNames[commType_] << endl; << " commsType=";
if (UPstream::parRun())
{
if (gatherv_) Info<< "gatherv+";
Info<< UPstream::commsTypeNames[commType_];
}
else
{
Info<< "serial";
}
Info<< endl;
} }
} }
@ -605,14 +620,29 @@ Foam::tmp<Foam::Field<Type>> Foam::surfaceWriter::mergeFieldTemplate
: mergedSurf_.faceGlobalIndex() : mergedSurf_.faceGlobalIndex()
); );
globIndex.gather if (gatherv_)
( {
fld, globIndex.mpiGather
allFld, (
UPstream::msgType(), fld,
commType_, allFld,
UPstream::worldComm UPstream::worldComm,
); // For fallback:
commType_,
UPstream::msgType()
);
}
else
{
globIndex.gather
(
fld,
allFld,
UPstream::msgType(),
commType_,
UPstream::worldComm
);
}
// Renumber (point data) to correspond to merged points // Renumber (point data) to correspond to merged points
if if
@ -626,6 +656,15 @@ Foam::tmp<Foam::Field<Type>> Foam::surfaceWriter::mergeFieldTemplate
allFld.resize(mergedSurf_.points().size()); allFld.resize(mergedSurf_.points().size());
} }
// Extended debugging. Limit to master:
#if 0
if (UPstream::master())
{
Info<< "merged List<" << pTraits<Type>::typeName << "> : ";
allFld.writeList(Info) << endl;
}
#endif
return tfield; return tfield;
} }

View File

@ -6,7 +6,7 @@
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2011-2012 OpenFOAM Foundation Copyright (C) 2011-2012 OpenFOAM Foundation
Copyright (C) 2015-2024 OpenCFD Ltd. Copyright (C) 2015-2025 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
@ -82,6 +82,7 @@ Description
Property | Description | Reqd | Default Property | Description | Reqd | Default
verbose | Additional output verbosity | no | no verbose | Additional output verbosity | no | no
commsType | Communication type | no | scheduled commsType | Communication type | no | scheduled
gatherv | Use MPI gatherv [experimental] | no | false
scale | Output geometry scaling | no | 1 scale | Output geometry scaling | no | 1
transform | Output coordinate transform | no | transform | Output coordinate transform | no |
fieldLevel | Subtract field level before scaling | no | empty dict fieldLevel | Subtract field level before scaling | no | empty dict
@ -97,6 +98,9 @@ Note
it is the responsibility of the implementation (not the caller) it is the responsibility of the implementation (not the caller)
to ensure that this occurs. to ensure that this occurs.
Using MPI gatherv [experimental] is not well tested and may change
or be removed in the future!
SourceFiles SourceFiles
surfaceWriter.C surfaceWriter.C
surfaceWriterI.H surfaceWriterI.H
@ -191,6 +195,9 @@ protected:
//- Communication type (for field merging) //- Communication type (for field merging)
UPstream::commsTypes commType_; UPstream::commsTypes commType_;
//- Prefer MPI gatherv intrinsic (for field merging) [experimental]
bool gatherv_;
//- The number of fields //- The number of fields
label nFields_; label nFields_;

View File

@ -88,8 +88,11 @@ Foam::surfaceWriters::debugWriter::debugWriter
streamOpt_(IOstreamOption::BINARY) streamOpt_(IOstreamOption::BINARY)
{ {
Info<< "Using debug surface writer (" Info<< "Using debug surface writer ("
<< (this->isPointData() ? "point" : "face") << " data):" << (this->isPointData() ? "point" : "face") << " data):";
<< " commsType=" << UPstream::commsTypeNames[commType_]
if (gatherv_) Info<< " <gatherv>";
Info<< " commsType=" << UPstream::commsTypeNames[commType_]
<< " merge=" << Switch::name(enableMerge_) << " merge=" << Switch::name(enableMerge_)
<< " write=" << Switch::name(enableWrite_) << endl; << " write=" << Switch::name(enableWrite_) << endl;
} }

View File

@ -0,0 +1,11 @@
#!/bin/sh
cd "${0%/*}" || exit # Run from this directory
. ${WM_PROJECT_DIR:?}/bin/tools/CleanFunctions # Tutorial clean functions
#------------------------------------------------------------------------------
cleanCase0
# Remove surface and features
rm -rf constant/triSurface
#------------------------------------------------------------------------------

View File

@ -49,7 +49,8 @@ __surfaceFieldValue
{ {
default default
{ {
verbose true; verbose true;
//gatherv true;
} }
} }

View File

@ -19,6 +19,12 @@ debug
formatOptions formatOptions
{ {
default
{
verbose true;
//gatherv true;
}
ensight ensight
{ {
collateTimes true; collateTimes true;