zhyang-dev/openfoam
1
0
Fork 0
mirror of https://develop.openfoam.com/Development/openfoam.git synced 2025-11-28 03:28:01 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

ENH: rationalize some globalIndex gather routines

Browse Source 
- replace gatherValues() with listGatherValues(), which returns the
  gathered values instead of passing by argument. This makes it less
  fragile and more convenient. Naming as per
  Pstream::listGatherValues(), but still retaining the original
  parameter order.

- rename inplace 'gather' variant as 'gatherInplace' for clarity.

- changed signature of lowest-level globalIndex::gather routines from
  List<Type> to UList<Type> for the output and removed any resizing.
  This means that the caller is responsible for ensuring that the
  receiving field is adequately sized (on master). This change allows
  potential reuse of 'work' arrays etc.

  The only code change for using this low-level gather was in
  GAMGAgglomeration.
This commit is contained in:
Mark Olesen
2025-02-06 18:45:39 +01:00
parent 3e8b0a2c73
commit 512ec01328
7 changed files with 364 additions and 290 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2019-2023 OpenCFD Ltd.
Copyright (C) 2019-2025 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -559,29 +559,28 @@ void Foam::GAMGAgglomeration::procAgglomerateRestrictAddressing
const label levelIndex
)
{
const bool master =
(
UPstream::myProcNo(comm) == (procIDs.empty() ? 0 : procIDs[0])
);
// Collect number of cells
labelList nFineCells;
globalIndex::gatherValues
labelList nFineCells = globalIndex::listGatherValues
(
comm,
procIDs,
restrictAddressing_[levelIndex].size(),
nFineCells,
UPstream::msgType(),
UPstream::commsTypes::scheduled
);
labelList fineOffsets(globalIndex::calcOffsets(nFineCells));
// Combine and renumber nCoarseCells
labelList nCoarseCells;
globalIndex::gatherValues
labelList nCoarseCells = globalIndex::listGatherValues
(
comm,
procIDs,
nCells_[levelIndex],
nCoarseCells,
UPstream::msgType(),
UPstream::commsTypes::scheduled
);
@ -589,6 +588,11 @@ void Foam::GAMGAgglomeration::procAgglomerateRestrictAddressing
// (cell)restrictAddressing
labelList procRestrictAddressing;
if (master)
{
// pre-size on master
procRestrictAddressing.resize(fineOffsets.back());
}
globalIndex::gather
(
fineOffsets,
@ -596,15 +600,13 @@ void Foam::GAMGAgglomeration::procAgglomerateRestrictAddressing
procIDs,
restrictAddressing_[levelIndex],
procRestrictAddressing,
UPstream::msgType(),
Pstream::commsTypes::nonBlocking //Pstream::commsTypes::scheduled
UPstream::commsTypes::nonBlocking
);
if (Pstream::myProcNo(comm) == procIDs[0])
if (master)
{
nCells_[levelIndex] = coarseOffsets.last(); // ie, totalSize()
nCells_[levelIndex] = coarseOffsets.back(); // ie, totalSize()
// Renumber consecutively
for (label proci = 1; proci < procIDs.size(); ++proci)

22
src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomerationTemplates.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2023 OpenCFD Ltd.
Copyright (C) 2023-2025 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -78,17 +78,17 @@ void Foam::GAMGAgglomeration::restrictField
const label coarseComm =
UPstream::parent(procCommunicator_[coarseLevelIndex]);
const List<label>& procIDs = agglomProcIDs(coarseLevelIndex);
const labelList& offsets = cellOffsets(coarseLevelIndex);
const auto& procIDs = agglomProcIDs(coarseLevelIndex);
const auto& offsets = cellOffsets(coarseLevelIndex);
globalIndex::gather
globalIndex::gatherInplace
(
offsets,
coarseComm,
procIDs,
cf,
UPstream::msgType(),
Pstream::commsTypes::nonBlocking //Pstream::commsTypes::scheduled
UPstream::commsTypes::nonBlocking
);
}
}
@ -145,8 +145,8 @@ void Foam::GAMGAgglomeration::prolongField
const label coarseComm =
UPstream::parent(procCommunicator_[coarseLevelIndex]);
const List<label>& procIDs = agglomProcIDs(coarseLevelIndex);
const labelList& offsets = cellOffsets(coarseLevelIndex);
const auto& procIDs = agglomProcIDs(coarseLevelIndex);
const auto& offsets = cellOffsets(coarseLevelIndex);
const label localSize = nCells_[levelIndex];
@ -159,7 +159,7 @@ void Foam::GAMGAgglomeration::prolongField
cf,
allCf,
UPstream::msgType(),
Pstream::commsTypes::nonBlocking //Pstream::commsTypes::scheduled
UPstream::commsTypes::nonBlocking
);
forAll(fineToCoarse, i)
@ -195,8 +195,8 @@ const Foam::Field<Type>& Foam::GAMGAgglomeration::prolongField
const label coarseComm =
UPstream::parent(procCommunicator_[coarseLevelIndex]);
const List<label>& procIDs = agglomProcIDs(coarseLevelIndex);
const labelList& offsets = cellOffsets(coarseLevelIndex);
const auto& procIDs = agglomProcIDs(coarseLevelIndex);
const auto& offsets = cellOffsets(coarseLevelIndex);
const label localSize = nCells_[levelIndex];
allCf.resize_nocopy(localSize);
@ -209,7 +209,7 @@ const Foam::Field<Type>& Foam::GAMGAgglomeration::prolongField
cf,
allCf,
UPstream::msgType(),
Pstream::commsTypes::nonBlocking //Pstream::commsTypes::scheduled
UPstream::commsTypes::nonBlocking
);
forAll(fineToCoarse, i)

134
src/OpenFOAM/parallel/globalIndex/globalIndex.H
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2018-2024 OpenCFD Ltd.
Copyright (C) 2018-2025 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -87,9 +87,8 @@ class globalIndex
// template<class Type>
// inline static UPstream::commsTypes getCommsType
// (
// const UPstream::commsTypes preferred
// = UPstream::commsTypes::nonBlocking
// );
// UPstream::commsTypes commsType
// ) noexcept;
//- Report overflow at specified (non-negative) index
static void reportOverflowAndExit
@ -597,19 +596,20 @@ public:
) const;
// Low-level gather routines
// Misc low-level gather routines
//- Collect single values in processor order on master (== procIDs[0]).
// Handles contiguous/non-contiguous data.
// non-zero output field (master only)
template<class ProcIDsContainer, class Type>
static void gatherValues
[[nodiscard]]
static List<Type> listGatherValues
(
const label comm, //!< communicator
const ProcIDsContainer& procIDs,
const Type& localValue,
List<Type>& allValues, //! output field (master only)
const int tag = UPstream::msgType(),
const UPstream::commsTypes = UPstream::commsTypes::nonBlocking
UPstream::commsTypes commsType = UPstream::commsTypes::nonBlocking
);
//- Collect data in processor order on master (== procIDs[0]).
@ -620,10 +620,11 @@ public:
const labelUList& offsets, //!< offsets (master only)
const label comm, //!< communicator
const ProcIDsContainer& procIDs,
const UList<Type>& fld,
List<Type>& allFld, //! output field (master only)
const UList<Type>& fld, //!< [in] all ranks
//! [out] result (master only). Must be adequately sized!
UList<Type>& allFld,
const int tag = UPstream::msgType(),
const UPstream::commsTypes = UPstream::commsTypes::nonBlocking
UPstream::commsTypes commsType = UPstream::commsTypes::nonBlocking
);
//- Collect indirect data in processor order on master
@ -634,37 +635,24 @@ public:
const labelUList& offsets, //!< offsets (master only)
const label comm, //!< communicator
const ProcIDsContainer& procIDs,
const IndirectListBase<Type, Addr>& fld,
List<Type>& allFld, //! output field (master only)
const IndirectListBase<Type, Addr>& fld, //!< [in] all ranks
//! [out] result (master only). Must be adequately sized!
UList<Type>& allFld,
const int tag = UPstream::msgType(),
const UPstream::commsTypes = UPstream::commsTypes::nonBlocking
UPstream::commsTypes commsType = UPstream::commsTypes::nonBlocking
);
// Misc low-level gather routines
//- Inplace collect in processor order on master (== procIDs[0]).
// Note: adjust naming?
template<class ProcIDsContainer, class Type>
static void gather
static void gatherInplace
(
const labelUList& offsets, //!< offsets (master only)
const label comm, //!< communicator
const ProcIDsContainer& procIDs,
List<Type>& fld, //!< in/out field
List<Type>& fld, //!< [in,out]
const int tag = UPstream::msgType(),
const UPstream::commsTypes ct = UPstream::commsTypes::nonBlocking
)
{
List<Type> allData;
gather(offsets, comm, procIDs, fld, allData, tag, ct);
const int masterProci = procIDs.size() ? procIDs[0] : 0;
if (UPstream::myProcNo(comm) == masterProci)
{
fld.transfer(allData);
}
}
UPstream::commsTypes commsType = UPstream::commsTypes::nonBlocking
);
//- Collect data in processor order on master (== procIDs[0]).
// \note the globalIndex offsets needed on master only.
@ -673,29 +661,26 @@ public:
(
const label comm, //!< communicator
const ProcIDsContainer& procIDs,
const UList<Type>& fld, //!< input field
List<Type>& allFld, //! output field (master only)
const UList<Type>& fld, //!< [in] input field
//! [out] resized to have results on master, empty elsewhere.
List<Type>& allFld,
const int tag = UPstream::msgType(),
const UPstream::commsTypes ct = UPstream::commsTypes::nonBlocking
) const
{
gather(offsets_, comm, procIDs, fld, allFld, tag, ct);
}
UPstream::commsTypes commsType = UPstream::commsTypes::nonBlocking
) const;
//- Inplace collect in processor order on master (== procIDs[0]).
// \note the globalIndex offsets needed on master only.
// Note: adjust naming?
template<class ProcIDsContainer, class Type>
void gather
void gatherInplace
(
const label comm, //!< communicator
const ProcIDsContainer& procIDs,
List<Type>& fld, //!< in/out field
List<Type>& fld, //!< [in,out]
const int tag = UPstream::msgType(),
const UPstream::commsTypes ct = UPstream::commsTypes::nonBlocking
UPstream::commsTypes commsType = UPstream::commsTypes::nonBlocking
) const
{
gather(offsets_, comm, procIDs, fld, tag, ct);
gatherInplace(offsets_, comm, procIDs, fld, tag, commsType);
}
@ -707,10 +692,12 @@ public:
template<class Type>
void gather
(
//! [in] input on all ranks
const UList<Type>& sendData,
//! [out] resized to have results on master, empty elsewhere.
List<Type>& allData,
const int tag = UPstream::msgType(),
const UPstream::commsTypes = UPstream::commsTypes::nonBlocking,
UPstream::commsTypes commsType = UPstream::commsTypes::nonBlocking,
const label comm = UPstream::worldComm //!< communicator
) const;
@ -720,10 +707,12 @@ public:
template<class Type, class Addr>
void gather
(
//! [in] input on all ranks
const IndirectListBase<Type, Addr>& sendData,
//! [out] resized to have results on master, empty elsewhere.
List<Type>& allData,
const int tag = UPstream::msgType(),
const UPstream::commsTypes = UPstream::commsTypes::nonBlocking,
UPstream::commsTypes commsType = UPstream::commsTypes::nonBlocking,
const label comm = UPstream::worldComm //!< communicator
) const;
@ -733,11 +722,12 @@ public:
//
// \return output (master), zero-sized on non-master
template<class Type, class OutputContainer = List<Type>>
[[nodiscard]]
OutputContainer gather
(
const UList<Type>& sendData,
const UList<Type>& sendData, //!< [in] all ranks
const int tag = UPstream::msgType(),
const UPstream::commsTypes = UPstream::commsTypes::nonBlocking,
UPstream::commsTypes commsType = UPstream::commsTypes::nonBlocking,
const label comm = UPstream::worldComm //!< communicator
) const;
@ -746,11 +736,12 @@ public:
//
// \return output (master), zero-sized on non-master
template<class Type, class Addr, class OutputContainer = List<Type>>
[[nodiscard]]
OutputContainer gather
(
const IndirectListBase<Type, Addr>& sendData,
const IndirectListBase<Type, Addr>& sendData, //!< [in] all ranks
const int tag = UPstream::msgType(),
const UPstream::commsTypes = UPstream::commsTypes::nonBlocking,
UPstream::commsTypes commsType = UPstream::commsTypes::nonBlocking,
const label comm = UPstream::worldComm //!< communicator
) const;
@ -761,10 +752,9 @@ public:
template<class Type>
void gatherInplace
(
//! [in,out]
List<Type>& fld,
List<Type>& fld, //!< [in,out]
const int tag = UPstream::msgType(),
const UPstream::commsTypes = UPstream::commsTypes::nonBlocking,
UPstream::commsTypes commsType = UPstream::commsTypes::nonBlocking,
const label comm = UPstream::worldComm //!< communicator
) const;
@ -782,7 +772,7 @@ public:
const label comm = UPstream::worldComm, //!< communicator
// For fallback routines:
const UPstream::commsTypes = UPstream::commsTypes::nonBlocking,
UPstream::commsTypes commsType = UPstream::commsTypes::nonBlocking,
const int tag = UPstream::msgType()
) const;
@ -794,13 +784,14 @@ public:
//
// \return output (master), zero-sized on non-master
template<class Type, class OutputContainer = List<Type>>
[[nodiscard]]
OutputContainer mpiGather
(
const UList<Type>& sendData,
const label comm = UPstream::worldComm, //!< communicator
// For fallback routines:
const UPstream::commsTypes = UPstream::commsTypes::nonBlocking,
UPstream::commsTypes commsType = UPstream::commsTypes::nonBlocking,
const int tag = UPstream::msgType()
) const;
@ -820,7 +811,7 @@ public:
const label comm = UPstream::worldComm, //!< communicator
// For fallback routines:
const UPstream::commsTypes = UPstream::commsTypes::nonBlocking,
UPstream::commsTypes commsType = UPstream::commsTypes::nonBlocking,
const int tag = UPstream::msgType()
) const;
@ -840,7 +831,7 @@ public:
const label comm = UPstream::worldComm, //!< communicator
// For fallback routines:
const UPstream::commsTypes = UPstream::commsTypes::nonBlocking,
UPstream::commsTypes commsType = UPstream::commsTypes::nonBlocking,
const int tag = UPstream::msgType()
);
@ -850,13 +841,14 @@ public:
//
// \return output (master), zero-sized on non-master
template<class Type, class OutputContainer = List<Type>>
[[nodiscard]]
static OutputContainer mpiGatherOp
(
const UList<Type>& sendData,
const label comm = UPstream::worldComm, //!< communicator
// For fallback routines:
const UPstream::commsTypes = UPstream::commsTypes::nonBlocking,
UPstream::commsTypes commsType = UPstream::commsTypes::nonBlocking,
const int tag = UPstream::msgType()
);
@ -874,7 +866,7 @@ public:
const label comm = UPstream::worldComm, //!< communicator
// For fallback routines:
const UPstream::commsTypes = UPstream::commsTypes::nonBlocking,
UPstream::commsTypes commsType = UPstream::commsTypes::nonBlocking,
const int tag = UPstream::msgType()
);
@ -888,7 +880,7 @@ public:
//! [out] output on master, zero-sized on non-master
List<Type>& allData,
const int tag = UPstream::msgType(),
const UPstream::commsTypes = UPstream::commsTypes::nonBlocking,
UPstream::commsTypes commsType = UPstream::commsTypes::nonBlocking,
const label comm = UPstream::worldComm //!< communicator
);
@ -902,7 +894,7 @@ public:
//! [out] output on master, zero-sized on non-master
List<Type>& allData,
const int tag = UPstream::msgType(),
const UPstream::commsTypes = UPstream::commsTypes::nonBlocking,
UPstream::commsTypes commsType = UPstream::commsTypes::nonBlocking,
const label comm = UPstream::worldComm //!< communicator
);
@ -912,11 +904,12 @@ public:
//
// \return output (master), zero-sized on non-master
template<class Type, class OutputContainer = List<Type>>
[[nodiscard]]
static OutputContainer gatherOp
(
const UList<Type>& sendData,
const int tag = UPstream::msgType(),
const UPstream::commsTypes = UPstream::commsTypes::nonBlocking,
UPstream::commsTypes commsType = UPstream::commsTypes::nonBlocking,
const label comm = UPstream::worldComm //!< communicator
);
@ -926,11 +919,12 @@ public:
//
// \return output (master), zero-sized on non-master
template<class Type, class Addr, class OutputContainer = List<Type>>
[[nodiscard]]
static OutputContainer gatherOp
(
const IndirectListBase<Type, Addr>& sendData,
const int tag = UPstream::msgType(),
const UPstream::commsTypes = UPstream::commsTypes::nonBlocking,
UPstream::commsTypes commsType = UPstream::commsTypes::nonBlocking,
const label comm = UPstream::worldComm //!< communicator
);
@ -945,7 +939,7 @@ public:
//! [in,out]
List<Type>& fld,
const int tag = UPstream::msgType(),
const UPstream::commsTypes = UPstream::commsTypes::nonBlocking,
UPstream::commsTypes commsType = UPstream::commsTypes::nonBlocking,
const label comm = UPstream::worldComm //!< communicator
);
@ -964,7 +958,7 @@ public:
const UList<Type>& allFld,
UList<Type>& fld,
const int tag = UPstream::msgType(),
const UPstream::commsTypes = UPstream::commsTypes::nonBlocking
UPstream::commsTypes commsType = UPstream::commsTypes::nonBlocking
);
//- Distribute data in processor order.
@ -979,11 +973,10 @@ public:
const UList<Type>& allFld,
UList<Type>& fld,
const int tag = UPstream::msgType(),
const UPstream::commsTypes ct =
UPstream::commsTypes::nonBlocking
UPstream::commsTypes commsType = UPstream::commsTypes::nonBlocking
) const
{
scatter(offsets_, comm, procIDs, allFld, fld, tag, ct);
scatter(offsets_, comm, procIDs, allFld, fld, tag, commsType);
}
//- Distribute data in processor order.
@ -996,7 +989,7 @@ public:
const UList<Type>& allData,
UList<Type>& localData,
const int tag = UPstream::msgType(),
const UPstream::commsTypes = UPstream::commsTypes::nonBlocking,
UPstream::commsTypes commsType = UPstream::commsTypes::nonBlocking,
const label comm = UPstream::worldComm //!< communicator
) const;
@ -1005,11 +998,12 @@ public:
// Communication with default/specified communicator, message tag.
// \note the globalIndex offsets needed on master only.
template<class Type, class OutputContainer = List<Type>>
[[nodiscard]]
OutputContainer scatter
(
const UList<Type>& allData,
const int tag = UPstream::msgType(),
const UPstream::commsTypes = UPstream::commsTypes::nonBlocking,
UPstream::commsTypes commsType = UPstream::commsTypes::nonBlocking,
const label comm = UPstream::worldComm //!< communicator
) const;

402
src/OpenFOAM/parallel/globalIndex/globalIndexTemplates.C
View File

@ -32,22 +32,22 @@ License
// Cannot use non-blocking for non-contiguous data.
// template<class Type>
// inline Foam::UPstream::commsTypes getCommsType
// inline Foam::UPstream::commsTypes Foam::globalIndex::getCommsType
// (
// const UPstream::commsTypes preferred
// )
// UPstream::commsTypes commsType
// ) noexcept
// {
// return
// (
// (
// !is_contiguous_v<Type>
// && UPstream::commsTypes::nonBlocking == preferred
// )
// ? UPstream::commsTypes::scheduled
// : preferred
// );
// if constexpr (!is_contiguous_v<Type>)
// {
// return UPstream::commsTypes::scheduled;
// }
// else
// {
// return commsType;
// }
// }
// Helpers
template<class Addr>
Foam::labelList
@ -121,34 +121,39 @@ Foam::globalIndex::calcListOffsets
}
// Low-level
template<class ProcIDsContainer, class Type>
void Foam::globalIndex::gatherValues
Foam::List<Type> Foam::globalIndex::listGatherValues
(
const label comm,
const ProcIDsContainer& procIDs,
const Type& localValue,
List<Type>& allValues,
const int tag,
const UPstream::commsTypes preferredCommsType
UPstream::commsTypes commsType
)
{
// low-level: no parRun guard
// low-level: no parRun guard?
const int masterProci = (procIDs.empty() ? 0 : procIDs[0]);
List<Type> allValues;
// if (!UPstream::is_parallel(comm))
// {
// allValues.resize(1);
// allValues[0] = localValue;
// return allValues;
// }
// Cannot use non-blocking for non-contiguous data
if constexpr (!is_contiguous_v<Type>)
{
commsType = UPstream::commsTypes::scheduled;
}
// Cannot use non-blocking for non-contiguous data.
const UPstream::commsTypes commsType =
(
(
!is_contiguous_v<Type>
&& UPstream::commsTypes::nonBlocking == preferredCommsType
)
? UPstream::commsTypes::scheduled
: preferredCommsType
);
const label startOfRequests = UPstream::nRequests();
const int masterProci = procIDs.size() ? procIDs[0] : 0;
if (UPstream::myProcNo(comm) == masterProci)
{
allValues.resize_nocopy(procIDs.size());
@ -176,8 +181,6 @@ void Foam::globalIndex::gatherValues
}
else
{
allValues.clear(); // safety: zero-size on non-master
if constexpr (is_contiguous_v<Type>)
{
UOPstream::write
@ -196,11 +199,10 @@ void Foam::globalIndex::gatherValues
}
}
if (commsType == UPstream::commsTypes::nonBlocking)
{
// Wait for outstanding requests
// Process sync
UPstream::waitRequests(startOfRequests);
}
return allValues;
}
@ -211,39 +213,31 @@ void Foam::globalIndex::gather
const label comm,
const ProcIDsContainer& procIDs,
const UList<Type>& fld,
List<Type>& allFld,
UList<Type>& allFld, // must be adequately sized on master
const int tag,
const UPstream::commsTypes preferredCommsType
UPstream::commsTypes commsType
)
{
// low-level: no parRun guard
const int masterProci = (procIDs.empty() ? 0 : procIDs[0]);
// Cannot use non-blocking for non-contiguous data.
const UPstream::commsTypes commsType =
(
(
!is_contiguous_v<Type>
&& UPstream::commsTypes::nonBlocking == preferredCommsType
)
? UPstream::commsTypes::scheduled
: preferredCommsType
);
// Cannot use non-blocking for non-contiguous data
if constexpr (!is_contiguous_v<Type>)
{
commsType = UPstream::commsTypes::scheduled;
}
const label startOfRequests = UPstream::nRequests();
const int masterProci = procIDs.size() ? procIDs[0] : 0;
if (UPstream::myProcNo(comm) == masterProci)
{
allFld.resize_nocopy(off.back()); // == totalSize()
// Assign my local data - respect offset information
// so that we can request 0 entries to be copied.
// Also handle the case where we have a slice of the full
// list.
SubList<Type>(allFld, off[1]-off[0], off[0]) =
SubList<Type>(fld, off[1]-off[0]);
if (FOAM_UNLIKELY(allFld.size() < off.back())) // ie, totalSize()
{
FatalErrorInFunction
<< "[out] UList size=" << allFld.size()
<< " too small to receive " << off.back() << nl
<< Foam::abort(FatalError);
}
for (label i = 1; i < procIDs.size(); ++i)
{
@ -269,6 +263,20 @@ void Foam::globalIndex::gather
IPstream::recv(procSlot, procIDs[i], tag, comm);
}
}
// Assign my local data - respect offset information
// so that we can request 0 entries to be copied.
// Also handle the case where we have a slice of the full
// list.
{
SubList<Type> dst(allFld, off[1]-off[0], off[0]);
SubList<Type> src(fld, off[1]-off[0]);
if (!dst.empty() && (dst.data() != src.data()))
{
dst = src;
}
}
}
else
{
@ -293,12 +301,9 @@ void Foam::globalIndex::gather
}
}
if (commsType == UPstream::commsTypes::nonBlocking)
{
// Wait for outstanding requests
// Process sync
UPstream::waitRequests(startOfRequests);
}
}
template<class ProcIDsContainer, class Type, class Addr>
@ -308,60 +313,52 @@ void Foam::globalIndex::gather
const label comm,
const ProcIDsContainer& procIDs,
const IndirectListBase<Type, Addr>& fld,
List<Type>& allFld,
UList<Type>& allFld, // must be adequately sized on master
const int tag,
const UPstream::commsTypes preferredCommsType
UPstream::commsTypes commsType
)
{
// low-level: no parRun guard
const int masterProci = (procIDs.empty() ? 0 : procIDs[0]);
if constexpr (is_contiguous_v<Type>)
{
// Flatten list (locally) so that we can benefit from using direct
// read/write of contiguous data
if (commsType == UPstream::commsTypes::nonBlocking)
{
// Contiguous data and requested nonBlocking.
//
// Flatten list (locally) so that we can benefit from using
// direct read/write of contiguous data
List<Type> flattened(fld);
gather
(
off,
comm,
procIDs,
List<Type>(fld),
flattened,
allFld,
tag,
preferredCommsType
commsType
);
return;
}
}
// Cannot use non-blocking for non-contiguous data.
const UPstream::commsTypes commsType =
(
(
!is_contiguous_v<Type>
&& UPstream::commsTypes::nonBlocking == preferredCommsType
)
? UPstream::commsTypes::scheduled
: preferredCommsType
);
const label startOfRequests = UPstream::nRequests();
const int masterProci = procIDs.size() ? procIDs[0] : 0;
// Non-contiguous is always non-blocking
if (UPstream::myProcNo(comm) == masterProci)
{
allFld.resize_nocopy(off.back()); // == totalSize()
// Assign my local data - respect offset information
// so that we can request 0 entries to be copied
SubList<Type> localSlot(allFld, off[1]-off[0], off[0]);
if (!localSlot.empty())
if (FOAM_UNLIKELY(allFld.size() < off.back())) // ie, totalSize()
{
localSlot = fld;
FatalErrorInFunction
<< "[out] UList size=" << allFld.size()
<< " too small to receive " << off.back() << nl
<< Foam::abort(FatalError);
}
// Already verified commsType != nonBlocking
for (label i = 1; i < procIDs.size(); ++i)
{
SubList<Type> procSlot(allFld, off[i+1]-off[i], off[i]);
@ -375,6 +372,17 @@ void Foam::globalIndex::gather
IPstream::recv(procSlot, procIDs[i], tag, comm);
}
}
// Assign my local data - respect offset information
// so that we can request 0 entries to be copied
{
SubList<Type> dst(allFld, off[1]-off[0], off[0]);
if (!dst.empty() && (dst.size() == fld.size()))
{
dst.deepCopy(fld);
}
}
}
else
{
@ -384,15 +392,81 @@ void Foam::globalIndex::gather
}
else
{
OPstream::send(fld, commsType, masterProci, tag, comm);
OPstream::send(fld, masterProci, tag, comm);
}
}
}
if (commsType == UPstream::commsTypes::nonBlocking)
template<class ProcIDsContainer, class Type>
void Foam::globalIndex::gatherInplace
(
const labelUList& off, // needed on master only
const label comm,
const ProcIDsContainer& procIDs,
List<Type>& fld,
const int tag,
UPstream::commsTypes commsType
)
{
// Wait for outstanding requests
UPstream::waitRequests(startOfRequests);
if (!UPstream::is_parallel(comm))
{
// Serial: (no-op)
return;
}
const bool master =
(
UPstream::myProcNo(comm) == (procIDs.empty() ? 0 : procIDs[0])
);
List<Type> allData;
if (master)
{
allData.resize_nocopy(off.back()); // == totalSize()
}
globalIndex::gather(off, comm, procIDs, fld, allData, tag, commsType);
if (master)
{
fld = std::move(allData);
}
else
{
fld.clear(); // zero-size on non-master
}
}
template<class ProcIDsContainer, class Type>
void Foam::globalIndex::gather
(
const label comm,
const ProcIDsContainer& procIDs,
const UList<Type>& fld,
List<Type>& allData,
const int tag,
UPstream::commsTypes commsType
) const
{
if (!UPstream::is_parallel(comm))
{
// Serial: (no-op)
return;
}
if (UPstream::myProcNo(comm) == (procIDs.empty() ? 0 : procIDs[0]))
{
// presize => totalSize()
allData.resize_nocopy(offsets_.back());
}
else
{
allData.clear(); // zero-size on non-master
}
globalIndex::gather(offsets_, comm, procIDs, fld, allData, tag, commsType);
}
@ -404,7 +478,7 @@ void Foam::globalIndex::gather
const UList<Type>& sendData,
List<Type>& allData,
const int tag,
const UPstream::commsTypes commsType,
UPstream::commsTypes commsType,
const label comm
) const
{
@ -415,6 +489,15 @@ void Foam::globalIndex::gather
return;
}
if (UPstream::master(comm))
{
allData.resize_nocopy(offsets_.back()); // == totalSize()
}
else
{
allData.clear(); // zero-size on non-master
}
{
globalIndex::gather
(
@ -426,10 +509,6 @@ void Foam::globalIndex::gather
tag,
commsType
);
if (!UPstream::master(comm))
{
allData.clear(); // safety: zero-size on non-master
}
}
}
@ -440,7 +519,7 @@ void Foam::globalIndex::gather
const IndirectListBase<Type, Addr>& sendData,
List<Type>& allData,
const int tag,
const UPstream::commsTypes commsType,
UPstream::commsTypes commsType,
const label comm
) const
{
@ -450,6 +529,37 @@ void Foam::globalIndex::gather
allData = sendData;
return;
}
else if constexpr (is_contiguous_v<Type>)
{
if (commsType == UPstream::commsTypes::nonBlocking)
{
// Contiguous data and requested nonBlocking.
//
// Flatten list (locally) so that we can benefit from using
// direct read/write of contiguous data
List<Type> flattened(sendData);
this->gather
(
flattened,
allData,
tag,
commsType,
comm
);
return;
}
}
if (UPstream::master(comm))
{
allData.resize_nocopy(offsets_.back()); // == totalSize()
}
else
{
allData.clear(); // zero-size on non-master
}
{
globalIndex::gather
@ -462,10 +572,6 @@ void Foam::globalIndex::gather
tag,
commsType
);
if (!UPstream::master(comm))
{
allData.clear(); // safety: zero-size on non-master
}
}
}
@ -475,12 +581,12 @@ OutputContainer Foam::globalIndex::gather
(
const UList<Type>& sendData,
const int tag,
const UPstream::commsTypes commsType,
UPstream::commsTypes commsType,
const label comm
) const
{
OutputContainer allData;
gather(sendData, allData, tag, commsType, comm);
this->gather(sendData, allData, tag, commsType, comm);
return allData;
}
@ -490,12 +596,12 @@ OutputContainer Foam::globalIndex::gather
(
const IndirectListBase<Type, Addr>& sendData,
const int tag,
const UPstream::commsTypes commsType,
UPstream::commsTypes commsType,
const label comm
) const
{
OutputContainer allData;
gather(sendData, allData, tag, commsType, comm);
this->gather(sendData, allData, tag, commsType, comm);
return allData;
}
@ -505,18 +611,18 @@ void Foam::globalIndex::gatherInplace
(
List<Type>& fld,
const int tag,
const UPstream::commsTypes commsType,
UPstream::commsTypes commsType,
const label comm
) const
{
if (UPstream::parRun())
{
List<Type> allData;
gather(fld, allData, tag, commsType, comm);
this->gather(fld, allData, tag, commsType, comm);
if (UPstream::master(comm))
{
fld.transfer(allData);
fld = std::move(allData);
}
else
{
@ -533,8 +639,7 @@ void Foam::globalIndex::mpiGather
const UList<Type>& sendData,
OutputContainer& allData,
const label comm,
const UPstream::commsTypes commsType,
UPstream::commsTypes commsType,
const int tag
) const
{
@ -696,8 +801,7 @@ OutputContainer Foam::globalIndex::mpiGather
(
const UList<Type>& sendData,
const label comm,
const UPstream::commsTypes commsType,
UPstream::commsTypes commsType,
const int tag
) const
{
@ -712,8 +816,7 @@ void Foam::globalIndex::mpiGatherInplace
(
List<Type>& fld,
const label comm,
const UPstream::commsTypes commsType,
UPstream::commsTypes commsType,
const int tag
) const
{
@ -724,7 +827,7 @@ void Foam::globalIndex::mpiGatherInplace
if (UPstream::master(comm))
{
fld.transfer(allData);
fld = std::move(allData);
}
else
{
@ -741,8 +844,7 @@ void Foam::globalIndex::mpiGatherOp
const UList<Type>& sendData,
OutputContainer& allData,
const label comm,
const UPstream::commsTypes commsType,
UPstream::commsTypes commsType,
const int tag
)
{
@ -765,8 +867,7 @@ OutputContainer Foam::globalIndex::mpiGatherOp
(
const UList<Type>& sendData,
const label comm,
const UPstream::commsTypes commsType,
UPstream::commsTypes commsType,
const int tag
)
{
@ -781,8 +882,7 @@ void Foam::globalIndex::mpiGatherInplaceOp
(
List<Type>& fld,
const label comm,
const UPstream::commsTypes commsType,
UPstream::commsTypes commsType,
const int tag
)
{
@ -793,7 +893,7 @@ void Foam::globalIndex::mpiGatherInplaceOp
if (UPstream::master(comm))
{
fld.transfer(allData);
fld = std::move(allData);
}
else
{
@ -810,7 +910,7 @@ void Foam::globalIndex::gatherOp
const UList<Type>& sendData,
List<Type>& allData,
const int tag,
const UPstream::commsTypes commsType,
UPstream::commsTypes commsType,
const label comm
)
{
@ -834,7 +934,7 @@ void Foam::globalIndex::gatherOp
const IndirectListBase<Type, Addr>& sendData,
List<Type>& allData,
const int tag,
const UPstream::commsTypes commsType,
UPstream::commsTypes commsType,
const label comm
)
{
@ -857,7 +957,7 @@ OutputContainer Foam::globalIndex::gatherOp
(
const UList<Type>& sendData,
const int tag,
const UPstream::commsTypes commsType,
UPstream::commsTypes commsType,
const label comm
)
{
@ -872,7 +972,7 @@ OutputContainer Foam::globalIndex::gatherOp
(
const IndirectListBase<Type, Addr>& sendData,
const int tag,
const UPstream::commsTypes commsType,
UPstream::commsTypes commsType,
const label comm
)
{
@ -887,7 +987,7 @@ void Foam::globalIndex::gatherInplaceOp
(
List<Type>& fld,
const int tag,
const UPstream::commsTypes commsType,
UPstream::commsTypes commsType,
const label comm
)
{
@ -910,26 +1010,21 @@ void Foam::globalIndex::scatter
const UList<Type>& allFld,
UList<Type>& fld,
const int tag,
const UPstream::commsTypes preferredCommsType
UPstream::commsTypes commsType
)
{
// low-level: no parRun guard
const int masterProci = (procIDs.empty() ? 0 : procIDs[0]);
// Cannot use non-blocking for non-contiguous data
if constexpr (!is_contiguous_v<Type>)
{
commsType = UPstream::commsTypes::scheduled;
}
// Cannot use non-blocking for non-contiguous data.
const UPstream::commsTypes commsType =
(
(
!is_contiguous_v<Type>
&& UPstream::commsTypes::nonBlocking == preferredCommsType
)
? UPstream::commsTypes::scheduled
: preferredCommsType
);
const label startOfRequests = UPstream::nRequests();
const int masterProci = procIDs.size() ? procIDs[0] : 0;
if (UPstream::myProcNo(comm) == masterProci)
{
for (label i = 1; i < procIDs.size(); ++i)
@ -962,8 +1057,15 @@ void Foam::globalIndex::scatter
// Also handle the case where we have a slice of the full
// list.
SubList<Type>(fld, off[1]-off[0]) =
SubList<Type>(allFld, off[1]-off[0], off[0]);
{
SubList<Type> dst(fld, off[1]-off[0]);
SubList<Type> src(allFld, off[1]-off[0], off[0]);
if (!dst.empty() && (dst.data() != src.data()))
{
dst = src;
}
}
}
else
{
@ -992,12 +1094,9 @@ void Foam::globalIndex::scatter
}
}
if (commsType == UPstream::commsTypes::nonBlocking)
{
// Wait for outstanding requests
// Process sync
UPstream::waitRequests(startOfRequests);
}
}
template<class Type>
@ -1006,7 +1105,7 @@ void Foam::globalIndex::scatter
const UList<Type>& allData,
UList<Type>& localData,
const int tag,
const UPstream::commsTypes commsType,
UPstream::commsTypes commsType,
const label comm
) const
{
@ -1037,7 +1136,7 @@ OutputContainer Foam::globalIndex::scatter
(
const UList<Type>& allData,
const int tag,
const UPstream::commsTypes commsType,
UPstream::commsTypes commsType,
const label comm
) const
{
@ -1051,7 +1150,8 @@ OutputContainer Foam::globalIndex::scatter
UPstream::listScatterValues<label>(this->localSizes(), comm)
);
OutputContainer localData(count);
OutputContainer localData;
localData.resize(count);
this->scatter(allData, localData, tag, commsType, comm);
return localData;

5
src/functionObjects/field/AMIWeights/AMIWeights.C
View File

@ -238,8 +238,7 @@ void Foam::functionObjects::AMIWeights::writeWeightField
);
// Collect field
scalarField mergedWeights;
globalFaces().gather(weightSum, mergedWeights);
scalarField mergedWeights = globalFaces().gather(weightSum);
const bool isACMI = isA<cyclicACMIPolyPatch>(cpp);
@ -248,7 +247,7 @@ void Foam::functionObjects::AMIWeights::writeWeightField
{
const cyclicACMIPolyPatch& pp = refCast<const cyclicACMIPolyPatch>(cpp);
globalFaces().gather(pp.mask(), mergedMask);
mergedMask = globalFaces().gather(pp.mask());
}
if (Pstream::master())

25
src/parallel/distributed/distributedTriSurfaceMesh/distributedTriSurfaceMesh.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2016 OpenFOAM Foundation
Copyright (C) 2015-2024 OpenCFD Ltd.
Copyright (C) 2015-2025 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -2441,12 +2441,7 @@ void Foam::distributedTriSurfaceMesh::independentlyDistributedBbs
// // Gather all borderTris
// //globalIndex globalBorderTris(borderTris.size());
// //pointField globalBorderCentres(allCentres, borderTris);
// //globalBorderTris.gather
// //(
// // UPstream::worldComm,
// // UPstream::allProcs(UPstream::worldComm),
// // globalBorderCentres
// //);
// //globalBorderTris.gatherInplace(globalBorderCentres);
// pointField globalBorderCentres(allCentres);
// map.distribute(globalBorderCentres);
//
@ -2586,12 +2581,7 @@ void Foam::distributedTriSurfaceMesh::independentlyDistributedBbs
{
allCentres[trii] = s[trii].centre(s.points());
}
globalTris().gather
(
UPstream::worldComm,
UPstream::allProcs(UPstream::worldComm),
allCentres
);
globalTris().gatherInplace(allCentres);
}
// Determine local decomposition
@ -2635,13 +2625,8 @@ void Foam::distributedTriSurfaceMesh::independentlyDistributedBbs
}
// Scatter back to processors
globalTris().scatter
(
UPstream::worldComm,
UPstream::allProcs(UPstream::worldComm),
allDistribution,
distribution
);
globalTris().scatter(allDistribution, distribution);
if (debug)
{
Pout<< "distributedTriSurfaceMesh::"

12
src/randomProcesses/noise/noiseModels/surfaceNoise/surfaceNoise.C
View File

@ -5,7 +5,7 @@
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2015-2023 OpenCFD Ltd.
Copyright (C) 2015-2025 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -274,12 +274,9 @@ scalar surfaceNoise::surfaceAverage
if (Pstream::parRun())
{
// Collect the surface data so that we can output the surfaces
scalarField allData;
procFaceAddr.gather
scalarField allData = procFaceAddr.gather
(
data,
allData,
UPstream::msgType(),
commType_,
UPstream::worldComm
@ -343,12 +340,9 @@ scalar surfaceNoise::writeSurfaceData
if (Pstream::parRun())
{
// Collect the surface data so that we can output the surfaces
scalarField allData;
procFaceAddr.gather
scalarField allData = procFaceAddr.gather
(
data,
allData,
UPstream::msgType(),
commType_,
UPstream::worldComm