zhyang-dev/openfoam
1
0
Fork 0
mirror of https://develop.openfoam.com/Development/openfoam.git synced 2025-11-28 03:28:01 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

ENH: AMIInterpolation: use local communicator

Browse Source 
- allocate communicator with only those ranks that have
patch faces
- use this communicator everywhere
- communicator preserved during mesh motion
This commit is contained in:
mattijs
2024-12-05 16:34:54 +00:00
parent f276016896
commit 34c933c1ce
25 changed files with 1884 additions and 1079 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
etc/controlDict
View File

@ -240,6 +240,12 @@ OptimisationSwitches
// from all the individual wall patches. Set to 0/false to revert to
// <v2412 behaviour
//useCombinedWallPatch 0;
//- Use local communicator for AMI communication. Default for v2506.
//localAMIComm = 0; // old behaviour : all ranks included
//localAMIComm = 1; // (default) only ranks that have patch faces
//localAMIComm = 2; // like 1 but always include rank 0 so messages
// still come from processor0
}

4
src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C
View File

@ -328,7 +328,7 @@ void Foam::GAMGAgglomeration::agglomerateLduAddressing
tmp<labelField> restrictMapInternalField;
// The finest mesh uses patchAddr from the original lduAdressing.
// the coarser levels create thei own adressing for faceCells
// the coarser levels create their own adressing for faceCells
if (fineLevelIndex == 0)
{
restrictMapInternalField =
@ -466,7 +466,7 @@ void Foam::GAMGAgglomeration::procAgglomerateLduAddressing
procBoundaryFaceMap_.set(levelIndex, new labelListListList(0));
// Collect meshes
// Collect meshes. Does no renumbering
PtrList<lduPrimitiveMesh> otherMeshes;
lduPrimitiveMesh::gather(comm, myMesh, otherMeshes);

4
src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/masterCoarsestGAMGProcAgglomeration/masterCoarsestGAMGProcAgglomeration.C
View File

@ -182,8 +182,8 @@ bool Foam::masterCoarsestGAMGProcAgglomeration::agglomerate()
for (const auto& p : masterToProcs)
{
Info<< '\t' << p[0]
<< '\t' << p.size()
<< '\t'
<< "\t\t" << p.size()
<< "\t\t"
<< flatOutput(SubList<label>(p, p.size()-1, 1))
<< endl;
}

268
src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeBase.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2015-2017 OpenFOAM Foundation
Copyright (C) 2015-2024 OpenCFD Ltd.
Copyright (C) 2015-2025 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -1001,17 +1001,39 @@ Foam::mapDistributeBase::mapDistributeBase
constructSize_(0),
subHasFlip_(false),
constructHasFlip_(false),
comm_(-1),
comm_(newComm),
schedulePtr_(nullptr)
{
if (maps.empty())
// localRanks : gives for every map the index (=rank in original
// communicator) in subMap,constructMap that refers to
// the local data. -1 if the map did not contain any data
// (can happen for e.g. agglomerating of cyclicAMI data)
// newToOldRanks : gives for every rank in the newComm the ranks in the
// maps that agglomerate to it. This is used to decide
// - data was local and stays local
// - data that was remote and now becomes local
// - and same for remote
// Currently can contain -1 entries. Probably should be
// filtered...
// Find set index
label validMapi = -1;
forAll(maps, mapi)
{
if (maps.set(mapi) && localRanks[mapi] != -1)
{
validMapi = mapi;
break;
}
}
if (validMapi == -1)
{
return;
}
comm_ = newComm;
subHasFlip_ = maps[0].subHasFlip();
constructHasFlip_ = maps[0].constructHasFlip();
subHasFlip_ = maps[validMapi].subHasFlip();
constructHasFlip_ = maps[validMapi].constructHasFlip();
const label nNewRanks = newToOldRanks.size();
const label myNewRank = UPstream::myProcNo(newComm);
@ -1031,15 +1053,20 @@ Foam::mapDistributeBase::mapDistributeBase
}
// Sanity checks
const auto& map0 = maps[0];
const auto& map0 = maps[validMapi];
forAll(maps, mapi)
{
if (!maps.set(mapi) || localRanks[mapi] == -1)
{
continue;
}
const auto& map = maps[mapi];
if
(
(map.comm() != map0.comm())
|| (map.subHasFlip() != map0.subHasFlip())
//(map.comm() != map0.comm())
(map.subHasFlip() != map0.subHasFlip())
|| (map.constructHasFlip() != map0.constructHasFlip())
)
{
@ -1049,7 +1076,10 @@ Foam::mapDistributeBase::mapDistributeBase
<< " has comm:" << map.comm()
<< " subHasFlip:" << map.subHasFlip()
<< " constructHasFlip:" << map.constructHasFlip()
<< " which is different from map 0"
<< " which is different from map 0 "
<< " comm:" << map0.comm()
<< " subHasFlip:" << map0.subHasFlip()
<< " constructHasFlip:" << map0.constructHasFlip()
<< exit(FatalError);
}
@ -1069,6 +1099,38 @@ Foam::mapDistributeBase::mapDistributeBase
}
// Determine start of constructed remote data. This is used to get the
// local offset which can then be used to get the relative subMap location.
labelListList startOfRemote(maps.size());
forAll(maps, mapi)
{
if (!maps.set(mapi))
{
continue;
}
const label nOldRanks = maps[mapi].constructMap().size();
labelList& starts = startOfRemote[mapi];
starts.setSize(nOldRanks, labelMax);
forAll(maps[mapi].constructMap(), oldRanki)
{
const labelList& map = maps[mapi].constructMap()[oldRanki];
forAll(map, i)
{
const label index
(
constructHasFlip_
? mag(map[i])-1
: map[i]
);
starts[oldRanki] = min(starts[oldRanki], index);
}
}
}
constructMap_.resize_nocopy(nNewRanks);
subMap_.resize_nocopy(nNewRanks);
@ -1081,7 +1143,14 @@ Foam::mapDistributeBase::mapDistributeBase
forAll(maps, mapi)
{
startOfLocal[mapi] = constructi;
const label localRank = localRanks[mapi];
if (!maps.set(mapi) || localRank == -1)
{
continue;
}
const auto& map = maps[mapi].constructMap()[localRank];
// Presize compaction array
@ -1093,31 +1162,6 @@ Foam::mapDistributeBase::mapDistributeBase
startOfLocal.last() = constructi;
// Determine start of constructed remote data. This is used to get the
// local offset which can then be used to get the relative subMap location.
labelListList startOfRemote(maps.size());
forAll(maps, mapi)
{
const label nOldProcs = maps[mapi].constructMap().size();
labelList& starts = startOfRemote[mapi];
starts.setSize(nOldProcs, labelMax);
forAll(maps[mapi].constructMap(), oldProci)
{
const labelList& map = maps[mapi].constructMap()[oldProci];
forAll(map, i)
{
const label index
(
constructHasFlip_
? mag(map[i])-1
: map[i]
);
starts[oldProci] = min(starts[oldProci], index);
}
}
}
// Construct map
// ~~~~~~~~~~~~~
@ -1141,6 +1185,12 @@ Foam::mapDistributeBase::mapDistributeBase
forAll(maps, mapi)
{
const label localRank = localRanks[mapi];
if (!maps.set(mapi) || localRank == -1)
{
continue;
}
const auto& map = maps[mapi].constructMap()[localRank];
const label offset = startOfLocal[mapi];
@ -1166,22 +1216,68 @@ Foam::mapDistributeBase::mapDistributeBase
}
}
}
if (constructi != startOfLocal.last())
{
FatalErrorInFunction
<< "constructi:" << constructi
<< " startOfLocal:" << startOfLocal.last()
<< exit(FatalError);
}
}
// Filter remote construct data
{
// Remote ranks that are now local
// - store new index for mapping stencils
// - no need to construct since already
const auto& oldProcs = newToOldRanks[myNewRank];
const auto& oldRanks = newToOldRanks[myNewRank];
// Determine number of old ranks. Note: should have passed in old-to-new
// ranks instead of new-to-old ranks?
label maxOldRank = -1;
for (const auto& elems : newToOldRanks)
{
for (const label el : elems)
{
maxOldRank = max(maxOldRank, el);
}
}
// Construct mapping from oldRanks to map. Note: could probably
// use some ListOps invert routine ...
labelList oldRankToMap(maxOldRank+1, -1);
{
forAll(localRanks, mapi)
{
const label localRank = localRanks[mapi];
if (localRank != -1)
{
oldRankToMap[localRank] = mapi;
}
}
}
forAll(maps, mapi)
{
for (const label oldProci : oldProcs)
if (!maps.set(mapi) || localRanks[mapi] == -1)
{
if (oldProci != localRanks[mapi])
continue;
}
for (const label oldRanki : oldRanks)
{
if (oldRanki == -1)
{
const auto& map = maps[mapi].constructMap()[oldProci];
continue;
}
if (oldRanki != localRanks[mapi])
{
// Note:
// - constructMap is sized with the map's communicator
const auto& map = maps[mapi].constructMap()[oldRanki];
if (!map.size())
{
@ -1191,11 +1287,11 @@ Foam::mapDistributeBase::mapDistributeBase
// The slots come from a local map so we can look up the
// new location
const label sourceMapi = localRanks.find(oldProci);
const label sourceMapi = oldRankToMap[oldRanki];
const auto& subMap =
maps[sourceMapi].subMap()[localRanks[mapi]];
//Pout<< "From oldRank:" << oldProci
//Pout<< "From oldRank:" << oldRanki
// << " sending to masterRank:" << localRanks[mapi]
// << " elements:" << flatOutput(subMap)
// << nl
@ -1205,7 +1301,7 @@ Foam::mapDistributeBase::mapDistributeBase
if (map.size() != subMap.size())
{
FatalErrorInFunction << "Problem:"
<< "oldProci:" << oldProci
<< "oldRanki:" << oldRanki
<< " mapi:" << mapi
<< " constructMap:" << map.size()
<< " sourceMapi:" << sourceMapi
@ -1215,7 +1311,7 @@ Foam::mapDistributeBase::mapDistributeBase
const label offset = startOfLocal[sourceMapi];
// Construct map starts after the local data
const label nMapLocal = startOfRemote[mapi][oldProci];
const label nMapLocal = startOfRemote[mapi][oldRanki];
auto& cptMap = compactMaps[mapi];
forAll(map, i)
@ -1238,7 +1334,7 @@ Foam::mapDistributeBase::mapDistributeBase
)
{
FatalErrorInFunction<< "Duplicate insertion"
<< "From oldProc:" << oldProci
<< "From oldRank:" << oldRanki
<< " on map:" << mapi
<< " at index:" << i
<< " have construct slot:" << index
@ -1260,32 +1356,51 @@ Foam::mapDistributeBase::mapDistributeBase
// Either loop over all old ranks and filter out ones already handled
// or loop over all new ranks and avoid myNewRank
forAll(newToOldRanks, newProci)
forAll(newToOldRanks, newRanki)
{
if (newProci != myNewRank)
if (newRanki != myNewRank)
{
const auto& oldProcs = newToOldRanks[newProci];
const auto& oldRanks = newToOldRanks[newRanki];
label allSize = 0;
forAll(maps, mapi)
{
for (const label oldProci : oldProcs)
if (!maps.set(mapi))
{
allSize += maps[mapi].constructMap()[oldProci].size();
continue;
}
for (const label oldRanki : oldRanks)
{
if (oldRanki == -1)
{
continue;
}
allSize += maps[mapi].constructMap()[oldRanki].size();
}
}
labelList& myConstruct = constructMap_[newProci];
labelList& myConstruct = constructMap_[newRanki];
myConstruct.resize_nocopy(allSize);
allSize = 0;
forAll(maps, mapi)
{
for (const label oldProci : oldProcs)
if (!maps.set(mapi))
{
const auto& map = maps[mapi].constructMap()[oldProci];
continue;
}
for (const label oldRanki : oldRanks)
{
if (oldRanki == -1)
{
continue;
}
const auto& map = maps[mapi].constructMap()[oldRanki];
// Construct map starts after the local data
const label nMapLocal = startOfRemote[mapi][oldProci];
const label nMapLocal = startOfRemote[mapi][oldRanki];
SubList<label> slice(myConstruct, map.size(), allSize);
if (constructHasFlip_)
@ -1340,6 +1455,12 @@ Foam::mapDistributeBase::mapDistributeBase
forAll(maps, mapi)
{
const label localRank = localRanks[mapi];
if (!maps.set(mapi) || localRank == -1)
{
continue;
}
allSize += maps[mapi].subMap()[localRank].size();
}
@ -1349,6 +1470,12 @@ Foam::mapDistributeBase::mapDistributeBase
forAll(maps, mapi)
{
const label localRank = localRanks[mapi];
if (!maps.set(mapi) || localRank == -1)
{
continue;
}
const auto& map = maps[mapi].subMap()[localRank];
SubList<label> slice(mySub, map.size(), allSize);
@ -1377,30 +1504,49 @@ Foam::mapDistributeBase::mapDistributeBase
}
}
// Filter remote sub data
forAll(newToOldRanks, newProci)
forAll(newToOldRanks, newRanki)
{
if (newProci != myNewRank)
if (newRanki != myNewRank)
{
const auto& oldProcs = newToOldRanks[newProci];
const auto& oldRanks = newToOldRanks[newRanki];
label allSize = 0;
forAll(maps, mapi)
{
for (const label oldProci : oldProcs)
if (!maps.set(mapi))
{
allSize += maps[mapi].subMap()[oldProci].size();
continue;
}
for (const label oldRanki : oldRanks)
{
if (oldRanki == -1)
{
continue;
}
allSize += maps[mapi].subMap()[oldRanki].size();
}
}
labelList& mySub = subMap_[newProci];
labelList& mySub = subMap_[newRanki];
mySub.resize_nocopy(allSize);
allSize = 0;
for (const label oldProci : oldProcs)
for (const label oldRanki : oldRanks)
{
if (oldRanki == -1)
{
continue;
}
forAll(maps, mapi)
{
const auto& map = maps[mapi].subMap()[oldProci];
if (!maps.set(mapi))
{
continue;
}
const auto& map = maps[mapi].subMap()[oldRanki];
SubList<label> slice(mySub, map.size(), allSize);
if (subHasFlip_)
{

22
src/finiteVolume/fields/fvPatchFields/constraint/cyclicACMI/cyclicACMIFvPatchField.C
View File

@ -345,7 +345,9 @@ template<class Type>
Foam::tmp<Foam::Field<Type>>
Foam::cyclicACMIFvPatchField<Type>::patchNeighbourField() const
{
if (this->ownerAMI().distributed() && cacheNeighbourField())
const auto& AMI = this->ownerAMI();
if (AMI.distributed() && cacheNeighbourField() && AMI.comm() != -1)
{
if (!this->ready())
{
@ -441,7 +443,9 @@ void Foam::cyclicACMIFvPatchField<Type>::initEvaluate
this->updateCoeffs();
}
if (this->ownerAMI().distributed() && cacheNeighbourField())
const auto& AMI = this->ownerAMI();
if (AMI.distributed() && cacheNeighbourField() && AMI.comm() != -1)
{
if (commsType != UPstream::commsTypes::nonBlocking)
{
@ -502,7 +506,7 @@ void Foam::cyclicACMIFvPatchField<Type>::evaluate
const auto& AMI = this->ownerAMI();
if (AMI.distributed() && cacheNeighbourField())
if (AMI.distributed() && cacheNeighbourField() && AMI.comm() != -1)
{
// Calculate patchNeighbourField
if (commsType != UPstream::commsTypes::nonBlocking)
@ -564,7 +568,9 @@ void Foam::cyclicACMIFvPatchField<Type>::initInterfaceMatrixUpdate
const Pstream::commsTypes commsType
) const
{
if (this->ownerAMI().distributed())
const auto& AMI = this->ownerAMI();
if (AMI.distributed() && AMI.comm() != -1)
{
// Start sending
if (commsType != UPstream::commsTypes::nonBlocking)
@ -640,7 +646,9 @@ void Foam::cyclicACMIFvPatchField<Type>::updateInterfaceMatrix
solveScalarField pnf;
if (this->ownerAMI().distributed())
const auto& AMI = this->ownerAMI();
if (AMI.distributed() && AMI.comm() != -1)
{
if (commsType != UPstream::commsTypes::nonBlocking)
{
@ -700,7 +708,7 @@ void Foam::cyclicACMIFvPatchField<Type>::initInterfaceMatrixUpdate
{
const auto& AMI = this->ownerAMI();
if (AMI.distributed())
if (AMI.distributed() && AMI.comm() != -1)
{
if (commsType != UPstream::commsTypes::nonBlocking)
{
@ -764,7 +772,7 @@ void Foam::cyclicACMIFvPatchField<Type>::updateInterfaceMatrix
Field<Type> pnf;
if (AMI.distributed())
if (AMI.distributed() && AMI.comm() != -1)
{
if (commsType != UPstream::commsTypes::nonBlocking)
{

20
src/finiteVolume/fields/fvPatchFields/constraint/cyclicAMI/cyclicAMIFvPatchField.C
View File

@ -338,7 +338,9 @@ template<class Type>
Foam::tmp<Foam::Field<Type>>
Foam::cyclicAMIFvPatchField<Type>::patchNeighbourField() const
{
if (this->ownerAMI().distributed() && cacheNeighbourField())
const auto& AMI = this->ownerAMI();
if (AMI.distributed() && cacheNeighbourField() && AMI.comm() != -1)
{
if (!this->ready())
{
@ -442,7 +444,9 @@ void Foam::cyclicAMIFvPatchField<Type>::initEvaluate
this->updateCoeffs();
}
if (this->ownerAMI().distributed() && cacheNeighbourField())
const auto& AMI = this->ownerAMI();
if (AMI.distributed() && cacheNeighbourField() && AMI.comm() != -1)
{
//DebugPout
// << "*** cyclicAMIFvPatchField::initEvaluate() :"
@ -506,7 +510,7 @@ void Foam::cyclicAMIFvPatchField<Type>::evaluate
const auto& AMI = this->ownerAMI();
if (AMI.distributed() && cacheNeighbourField())
if (AMI.distributed() && cacheNeighbourField() && AMI.comm() != -1)
{
// Calculate patchNeighbourField
if (commsType != UPstream::commsTypes::nonBlocking)
@ -574,7 +578,9 @@ void Foam::cyclicAMIFvPatchField<Type>::initInterfaceMatrixUpdate
const Pstream::commsTypes commsType
) const
{
if (this->ownerAMI().distributed())
const auto& AMI = this->ownerAMI();
if (AMI.distributed() && AMI.comm() != -1)
{
// Start sending
if (commsType != UPstream::commsTypes::nonBlocking)
@ -645,7 +651,7 @@ void Foam::cyclicAMIFvPatchField<Type>::updateInterfaceMatrix
solveScalarField pnf;
if (AMI.distributed())
if (AMI.distributed() && AMI.comm() != -1)
{
if (commsType != UPstream::commsTypes::nonBlocking)
{
@ -714,7 +720,7 @@ void Foam::cyclicAMIFvPatchField<Type>::initInterfaceMatrixUpdate
{
const auto& AMI = this->ownerAMI();
if (AMI.distributed())
if (AMI.distributed() && AMI.comm() != -1)
{
if (commsType != UPstream::commsTypes::nonBlocking)
{
@ -783,7 +789,7 @@ void Foam::cyclicAMIFvPatchField<Type>::updateInterfaceMatrix
Field<Type> pnf;
if (AMI.distributed())
if (AMI.distributed() && AMI.comm() != -1)
{
if (commsType != UPstream::commsTypes::nonBlocking)
{

2
src/finiteVolume/fvMesh/fvPatches/constraint/cyclicAMI/cyclicAMIFvPatch.C
View File

@ -289,7 +289,7 @@ void Foam::cyclicAMIFvPatch::movePoints()
}
scalarField srcMeshPhi(phip);
if (AMI().distributed())
if (AMI().distributed() && AMI().comm() != -1)
{
AMI().srcMap().distribute(srcMeshPhi);
}

240
src/meshTools/AMIInterpolation/AMIInterpolation/AMIInterpolation.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2015-2023 OpenCFD Ltd.
Copyright (C) 2015-2024 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -33,6 +33,7 @@ License
#include "profiling.H"
#include "triangle.H"
#include "OFstream.H"
#include "registerSwitch.H"
#include <numeric> // For std::iota
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
@ -46,6 +47,17 @@ namespace Foam
bool Foam::AMIInterpolation::cacheIntersections_ = false;
int Foam::AMIInterpolation::localComm_
(
debug::optimisationSwitch("localAMIComm", 1)
);
registerOptSwitch
(
"localAMIComm",
int,
Foam::AMIInterpolation::localComm_
);
// * * * * * * * * * * * * Protected Member Functions * * * * * * * * * * * //
@ -77,7 +89,9 @@ Foam::AMIInterpolation::createTree
Foam::label Foam::AMIInterpolation::calcDistribution
(
const primitivePatch& srcPatch,
const primitivePatch& tgtPatch
const primitivePatch& tgtPatch,
const label comm,
autoPtr<UPstream::communicator>& geomComm
) const
{
// Either not parallel or no faces on any processor
@ -85,12 +99,40 @@ Foam::label Foam::AMIInterpolation::calcDistribution
if (Pstream::parRun())
{
bool hasLocalFaces = false;
if (localComm_ == 0)
{
// Backwards compatible : all processors involved
hasLocalFaces = true;
}
else if (localComm_ == 1)
{
// Only if locally have faces
hasLocalFaces = (srcPatch.size() > 0 || tgtPatch.size() > 0);
}
else
{
// If master (so messages always come from master) or if locally
// have faces
hasLocalFaces =
(
UPstream::master(comm)
|| srcPatch.size() > 0
|| tgtPatch.size() > 0
);
}
// Better to use MPI_Comm_split? So only provide local information
// (hasLocalFaces). Problem is that we don't know who else is in the
// set. Whereas now, because we all loop over the same data in the same
// order we coud find out (except we don't use this information?)
const bitSet hasFaces
(
UPstream::listGatherValues<bool>
UPstream::allGatherValues<bool>
(
(srcPatch.size() > 0 || tgtPatch.size() > 0),
comm_
hasLocalFaces,
comm
)
);
@ -98,18 +140,48 @@ Foam::label Foam::AMIInterpolation::calcDistribution
if (nHaveFaces == 1)
{
// Release any previously allocated communicator
geomComm.clear();
proci = hasFaces.find_first();
DebugInFunction
<< "AMI local to processor" << proci << endl;
}
else if (nHaveFaces > 1)
{
if (hasLocalFaces)
{
geomComm.reset
(
new UPstream::communicator
(
comm,
hasFaces.sortedToc()
)
);
if (debug)
{
Pout<< "Allocated geomComm:" << geomComm()
<< " from " << nHaveFaces
<< " processors out of " << UPstream::nProcs(comm)
<< endl;
}
}
else
{
geomComm.reset(new UPstream::communicator());
if (debug & 2)
{
Pout<< "Allocated dummy geomComm:" << geomComm()
<< " since no src:" << srcPatch.size()
<< " and no tgt:" << tgtPatch.size() << endl;
}
}
proci = -1;
DebugInFunction
<< "AMI split across multiple processors" << endl;
<< "AMI split across multiple processors "
<< flatOutput(hasFaces.sortedToc()) << endl;
}
Pstream::broadcast(proci, comm_);
}
return proci;
@ -207,7 +279,7 @@ void Foam::AMIInterpolation::normaliseWeights
}
}
if (output)
if (output && comm != -1)
{
// Note: change global communicator since gMin,gAverage etc don't
// support user communicator
@ -216,7 +288,8 @@ void Foam::AMIInterpolation::normaliseWeights
if (returnReduceOr(wght.size()))
{
Info<< indent
Info.masterStream(comm)
<< indent
<< "AMI: Patch " << patchName
<< " sum(weights)"
<< " min:" << gMin(wghtSum)
@ -227,7 +300,8 @@ void Foam::AMIInterpolation::normaliseWeights
if (nLow)
{
Info<< indent
Info.masterStream(comm)
<< indent
<< "AMI: Patch " << patchName
<< " identified " << nLow
<< " faces with weights less than " << lowWeightTol
@ -260,14 +334,14 @@ void Foam::AMIInterpolation::agglomerate
{
addProfiling(ami, "AMIInterpolation::agglomerate");
label sourceCoarseSize =
const label sourceCoarseSize =
(
sourceRestrictAddressing.size()
? max(sourceRestrictAddressing)+1
: 0
);
label targetCoarseSize =
const label targetCoarseSize =
(
targetRestrictAddressing.size()
? max(targetRestrictAddressing)+1
@ -289,8 +363,22 @@ void Foam::AMIInterpolation::agglomerate
// Agglomerate weights and indices
if (targetMapPtr)
{
// We are involved in the communicator but our maps are still empty.
// Fix 'm up so they are the same size as the communicator.
const mapDistribute& map = *targetMapPtr;
if (map.constructMap().empty())
{
auto& cMap = const_cast<labelListList&>(map.constructMap());
cMap.resize_nocopy(UPstream::nProcs(map.comm()));
}
if (map.subMap().empty())
{
auto& cMap = const_cast<labelListList&>(map.subMap());
cMap.resize_nocopy(UPstream::nProcs(map.comm()));
}
// Get all restriction addressing.
labelList allRestrict(targetRestrictAddressing);
map.distribute(allRestrict);
@ -621,7 +709,8 @@ Foam::AMIInterpolation::AMIInterpolation
reverseTarget_(fineAMI.reverseTarget_),
lowWeightCorrection_(-1.0),
singlePatchProc_(fineAMI.singlePatchProc_),
comm_(fineAMI.comm_),
comm_(fineAMI.comm()), // use fineAMI geomComm if present, comm otherwise
geomComm_(),
srcMagSf_(),
srcAddress_(),
srcWeights_(),
@ -681,41 +770,79 @@ Foam::AMIInterpolation::AMIInterpolation
// Agglomerate addresses and weights
agglomerate
(
fineAMI.tgtMapPtr_,
fineAMI.srcMagSf(),
fineAMI.srcAddress(),
fineAMI.srcWeights(),
if (comm() != -1)
{
//Pout<< "** agglomerating srcAddress, tgtMap" << endl;
//if (fineAMI.tgtMapPtr_.valid())
//{
// const auto& fineTgtMap = fineAMI.tgtMapPtr_();
// Pout<< " fineAMI.tgtMapPtr_ comm:" << fineTgtMap.comm()
// << " procs:"
// << (
// fineTgtMap.comm() != -1
// ? UPstream::procID(fineTgtMap.comm())
// : labelList::null()
// )
// << endl;
//}
//else
//{
// Pout<< " NO fineAMI.tgtMapPtr_" << endl;
//}
//
agglomerate
(
fineAMI.tgtMapPtr_,
fineAMI.srcMagSf(),
fineAMI.srcAddress(),
fineAMI.srcWeights(),
sourceRestrictAddressing,
targetRestrictAddressing,
sourceRestrictAddressing,
targetRestrictAddressing,
srcMagSf_,
srcAddress_,
srcWeights_,
srcWeightsSum_,
tgtMapPtr_,
comm_
);
srcMagSf_,
srcAddress_,
srcWeights_,
srcWeightsSum_,
tgtMapPtr_,
comm()
);
agglomerate
(
fineAMI.srcMapPtr_,
fineAMI.tgtMagSf(),
fineAMI.tgtAddress(),
fineAMI.tgtWeights(),
//Pout<< "** agglomerating tgtAddress, srcMap" << endl;
//if (fineAMI.srcMapPtr_.valid())
//{
// const auto& fineSrcMap = fineAMI.srcMapPtr_();
// Pout<< " fineAMI.srcMapPtr_ comm:" << fineSrcMap.comm()
// << " procs:"
// << (
// fineSrcMap.comm() != -1
// ? UPstream::procID(fineSrcMap.comm())
// : labelList::null()
// )
// << endl;
//}
//else
//{
// Pout<< " NO fineAMI.srcMapPtr_" << endl;
//}
agglomerate
(
fineAMI.srcMapPtr_,
fineAMI.tgtMagSf(),
fineAMI.tgtAddress(),
fineAMI.tgtWeights(),
targetRestrictAddressing,
sourceRestrictAddressing,
targetRestrictAddressing,
sourceRestrictAddressing,
tgtMagSf_,
tgtAddress_,
tgtWeights_,
tgtWeightsSum_,
srcMapPtr_,
comm_
);
tgtMagSf_,
tgtAddress_,
tgtWeights_,
tgtWeightsSum_,
srcMapPtr_,
comm()
);
}
}
@ -726,6 +853,7 @@ Foam::AMIInterpolation::AMIInterpolation(const AMIInterpolation& ami)
lowWeightCorrection_(ami.lowWeightCorrection_),
singlePatchProc_(ami.singlePatchProc_),
comm_(ami.comm_),
geomComm_(ami.geomComm_), // ? steals communicator
srcMagSf_(ami.srcMagSf_),
srcAddress_(ami.srcAddress_),
srcWeights_(ami.srcWeights_),
@ -748,7 +876,7 @@ Foam::AMIInterpolation::AMIInterpolation(Istream& is)
reverseTarget_(readBool(is)),
lowWeightCorrection_(readScalar(is)),
singlePatchProc_(readLabel(is)),
comm_(readLabel(is)),
comm_(readLabel(is)), // either geomComm_ or comm_ from sending side
srcMagSf_(is),
srcAddress_(is),
@ -768,7 +896,10 @@ Foam::AMIInterpolation::AMIInterpolation(Istream& is)
upToDate_(readBool(is))
{
if (singlePatchProc_ == -1)
// Hopefully no need to stream geomComm_ since only used in processor
// agglomeration?
if (singlePatchProc_ == -1 && comm_ != -1)
{
srcMapPtr_.reset(new mapDistribute(is));
tgtMapPtr_.reset(new mapDistribute(is));
@ -816,6 +947,7 @@ bool Foam::AMIInterpolation::calculate
ttgtPatch0_.cref(tgtPatch);
}
// Note: use original communicator for statistics
label srcTotalSize = returnReduce
(
srcPatch.size(),
@ -842,7 +974,11 @@ bool Foam::AMIInterpolation::calculate
return false;
}
singlePatchProc_ = calcDistribution(srcPatch, tgtPatch);
// Calculate:
// - which processors have faces
// - allocates a communicator (geomComm_) for those
// - if it is only one processor that holds all faces
singlePatchProc_ = calcDistribution(srcPatch, tgtPatch, comm_, geomComm_);
Info<< indent << "AMI: Patch source faces: " << srcTotalSize << nl
<< indent << "AMI: Patch target faces: " << tgtTotalSize << nl;
@ -913,7 +1049,7 @@ void Foam::AMIInterpolation::append
newPtr->calculate(srcPatch, tgtPatch);
// If parallel then combine the mapDistribution and re-index
if (distributed())
if (distributed() && comm() != -1)
{
labelListList& srcSubMap = srcMapPtr_->subMap();
labelListList& srcConstructMap = srcMapPtr_->constructMap();
@ -1107,7 +1243,7 @@ void Foam::AMIInterpolation::normaliseWeights
conformal,
output,
lowWeightCorrection_,
comm_
comm()
);
normaliseWeights
@ -1120,7 +1256,7 @@ void Foam::AMIInterpolation::normaliseWeights
conformal,
output,
lowWeightCorrection_,
comm_
comm()
);
}
@ -1345,7 +1481,7 @@ bool Foam::AMIInterpolation::writeData(Ostream& os) const
<< token::SPACE<< reverseTarget()
<< token::SPACE<< lowWeightCorrection()
<< token::SPACE<< singlePatchProc()
<< token::SPACE<< comm()
<< token::SPACE<< comm() // either geomComm_ or comm_
<< token::SPACE<< srcMagSf()
<< token::SPACE<< srcAddress()
@ -1361,7 +1497,7 @@ bool Foam::AMIInterpolation::writeData(Ostream& os) const
<< token::SPACE<< upToDate();
if (distributed())
if (distributed() && comm() != -1)
{
os << token::SPACE<< srcMap()
<< token::SPACE<< tgtMap();

19
src/meshTools/AMIInterpolation/AMIInterpolation/AMIInterpolation.H
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2016-2023 OpenCFD Ltd.
Copyright (C) 2016-2024 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -80,6 +80,11 @@ public:
static bool cacheIntersections_;
//- localComm : 0 : all processors (backwards compatible)
//- localComm : 1 : only processors with patch faces
//- localComm : 2 : like 1 but add master processor always
static int localComm_;
protected:
@ -106,6 +111,10 @@ protected:
//- Communicator to use for parallel operations.
label comm_;
//- Communicator to use for geometry calculations. Not valid (-1) on
//- processors that do not have faces
autoPtr<UPstream::communicator> geomComm_;
// Source patch
@ -181,11 +190,15 @@ protected:
const primitivePatch& patch
) const;
//- Calculate if patches are on multiple processors
//- Calculate if patches are on multiple processors. Allocates
//- local communicator and returns -1 or processor containing all
//- faces
label calcDistribution
(
const primitivePatch& srcPatch,
const primitivePatch& tgtPatch
const primitivePatch& tgtPatch,
const label comm,
autoPtr<UPstream::communicator>& geomComm
) const;
//- Project points to surface

9
src/meshTools/AMIInterpolation/AMIInterpolation/AMIInterpolationI.H
View File

@ -104,7 +104,14 @@ inline Foam::label Foam::AMIInterpolation::singlePatchProc() const noexcept
inline Foam::label Foam::AMIInterpolation::comm() const
{
return comm_;
if (geomComm_.valid())
{
return geomComm_();
}
else
{
return comm_;
}
}

16
src/meshTools/AMIInterpolation/AMIInterpolation/AMIInterpolationTemplates.C
View File

@ -142,9 +142,15 @@ void Foam::AMIInterpolation::interpolateToTarget
result.setSize(tgtAddress_.size());
List<Type> work;
if (distributed())
if (distributed() && srcMapPtr_)
{
const mapDistribute& map = srcMapPtr_();
if (map.comm() == -1)
{
return;
}
work.resize_nocopy(map.constructSize());
SubList<Type>(work, fld.size()) = fld; // deep copy
map.distribute(work);
@ -203,9 +209,15 @@ void Foam::AMIInterpolation::interpolateToSource
result.setSize(srcAddress_.size());
List<Type> work;
if (distributed())
if (distributed() && tgtMapPtr_)
{
const mapDistribute& map = tgtMapPtr_();
if (map.comm() == -1)
{
return;
}
work.resize_nocopy(map.constructSize());
SubList<Type>(work, fld.size()) = fld; // deep copy
map.distribute(work);

16
src/meshTools/AMIInterpolation/AMIInterpolation/advancingFrontAMI/advancingFrontAMI.C
View File

@ -63,7 +63,7 @@ void Foam::advancingFrontAMI::checkPatches() const
}
if (requireMatch_)
if (requireMatch_ && comm() != -1)
{
const scalar maxBoundsError = 0.05;
@ -74,14 +74,14 @@ void Foam::advancingFrontAMI::checkPatches() const
bbSrc.min(),
minOp<point>(),
UPstream::msgType(),
comm_
comm()
);
Foam::reduce
(
bbSrc.max(),
maxOp<point>(),
UPstream::msgType(),
comm_
comm()
);
boundBox bbTgt(tgt.points(), tgt.meshPoints(), false);
Foam::reduce
@ -89,14 +89,14 @@ void Foam::advancingFrontAMI::checkPatches() const
bbTgt.min(),
minOp<point>(),
UPstream::msgType(),
comm_
comm()
);
Foam::reduce
(
bbTgt.max(),
maxOp<point>(),
UPstream::msgType(),
comm_
comm()
);
boundBox bbTgtInf(bbTgt);
@ -178,7 +178,7 @@ void Foam::advancingFrontAMI::createExtendedTgtPatch()
// Original faces from tgtPatch
// Note: in globalIndexing since might be remote
globalIndex globalTgtFaces(tgtPatch0().size(), comm_);
globalIndex globalTgtFaces(tgtPatch0().size(), comm());
distributeAndMergePatches
(
map,
@ -465,7 +465,7 @@ void Foam::advancingFrontAMI::triangulatePatch
void Foam::advancingFrontAMI::nonConformalCorrection()
{
if (!requireMatch_ && distributed())
if (!requireMatch_ && distributed() && comm() != -1)
{
scalarList newTgtMagSf(std::move(tgtMagSf_));
@ -587,7 +587,7 @@ bool Foam::advancingFrontAMI::calculate
{
// Create a representation of the target patch that covers the source
// patch
if (distributed())
if (distributed() && comm() != -1)
{
createExtendedTgtPatch();
}

53
src/meshTools/AMIInterpolation/AMIInterpolation/advancingFrontAMI/advancingFrontAMIParallelOps.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2018-2022 OpenCFD Ltd.
Copyright (C) 2018-2024 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -74,13 +74,13 @@ void Foam::advancingFrontAMI::distributePatches
List<labelList>& faceIDs
) const
{
faces.setSize(Pstream::nProcs(comm_));
points.setSize(Pstream::nProcs(comm_));
faceIDs.setSize(Pstream::nProcs(comm_));
faces.setSize(Pstream::nProcs(comm()));
points.setSize(Pstream::nProcs(comm()));
faceIDs.setSize(Pstream::nProcs(comm()));
PstreamBuffers pBufs(comm_);
PstreamBuffers pBufs(comm());
for (const int domain : Pstream::allProcs(comm_))
for (const int domain : Pstream::allProcs(comm()))
{
const labelList& sendElems = map.subMap()[domain];
@ -103,13 +103,13 @@ void Foam::advancingFrontAMI::distributePatches
}
if (domain == Pstream::myProcNo(comm_))
if (domain == Pstream::myProcNo(comm()))
{
// Do send/receive for myself
faces[domain] = subPatch.localFaces();
points[domain] = subPatch.localPoints();
faceIDs[domain] =
gi.toGlobal(Pstream::myProcNo(comm_), sendElems);
gi.toGlobal(Pstream::myProcNo(comm()), sendElems);
}
else
{
@ -118,7 +118,7 @@ void Foam::advancingFrontAMI::distributePatches
str
<< subPatch.localFaces()
<< subPatch.localPoints()
<< gi.toGlobal(Pstream::myProcNo(comm_), sendElems);
<< gi.toGlobal(Pstream::myProcNo(comm()), sendElems);
}
}
}
@ -127,11 +127,11 @@ void Foam::advancingFrontAMI::distributePatches
// Consume
for (const int domain : Pstream::allProcs(comm_))
for (const int domain : Pstream::allProcs(comm()))
{
const labelList& recvElems = map.constructMap()[domain];
if (domain != Pstream::myProcNo(comm_) && recvElems.size())
if (domain != Pstream::myProcNo(comm()) && recvElems.size())
{
UIPstream is(domain, pBufs);
@ -177,10 +177,10 @@ void Foam::advancingFrontAMI::distributeAndMergePatches
// My own data first
{
const labelList& faceIDs = allTgtFaceIDs[Pstream::myProcNo(comm_)];
const labelList& faceIDs = allTgtFaceIDs[Pstream::myProcNo(comm())];
SubList<label>(tgtFaceIDs, faceIDs.size()) = faceIDs;
const faceList& fcs = allFaces[Pstream::myProcNo(comm_)];
const faceList& fcs = allFaces[Pstream::myProcNo(comm())];
for (const face& f : fcs)
{
face& newF = tgtFaces[nFaces++];
@ -191,7 +191,7 @@ void Foam::advancingFrontAMI::distributeAndMergePatches
}
}
const pointField& pts = allPoints[Pstream::myProcNo(comm_)];
const pointField& pts = allPoints[Pstream::myProcNo(comm())];
for (const point& pt: pts)
{
tgtPoints[nPoints++] = pt;
@ -202,7 +202,7 @@ void Foam::advancingFrontAMI::distributeAndMergePatches
// Other proc data follows
forAll(allFaces, proci)
{
if (proci != Pstream::myProcNo(comm_))
if (proci != Pstream::myProcNo(comm()))
{
const labelList& faceIDs = allTgtFaceIDs[proci];
SubList<label>(tgtFaceIDs, faceIDs.size(), nFaces) = faceIDs;
@ -258,12 +258,19 @@ Foam::autoPtr<Foam::mapDistribute> Foam::advancingFrontAMI::calcProcMap
const primitivePatch& tgtPatch
) const
{
if (comm() == -1)
{
FatalErrorInFunction
<< "Processor " << UPstream::myProcNo(UPstream::worldComm)
<< " not in communicator " << comm() << exit(FatalError);
}
// Get decomposition of patch
List<treeBoundBoxList> procBb(Pstream::nProcs(comm_));
List<treeBoundBoxList> procBb(Pstream::nProcs(comm()));
if (srcPatch.size())
{
procBb[Pstream::myProcNo(comm_)] =
procBb[Pstream::myProcNo(comm())] =
AABBTree<face>
(
srcPatch.localFaces(),
@ -273,10 +280,10 @@ Foam::autoPtr<Foam::mapDistribute> Foam::advancingFrontAMI::calcProcMap
}
else
{
procBb[Pstream::myProcNo(comm_)] = treeBoundBoxList();
procBb[Pstream::myProcNo(comm())] = treeBoundBoxList();
}
Pstream::allGatherList(procBb, UPstream::msgType(), comm_);
Pstream::allGatherList(procBb, UPstream::msgType(), comm());
if (debug)
{
@ -296,10 +303,10 @@ Foam::autoPtr<Foam::mapDistribute> Foam::advancingFrontAMI::calcProcMap
{
// Per processor indices into all segments to send
List<DynamicList<label>> dynSendMap(Pstream::nProcs(comm_));
List<DynamicList<label>> dynSendMap(Pstream::nProcs(comm()));
// Work array - whether processor bb overlaps the face bounds
boolList procBbOverlaps(Pstream::nProcs(comm_));
boolList procBbOverlaps(Pstream::nProcs(comm()));
forAll(faces, facei)
{
@ -321,7 +328,7 @@ Foam::autoPtr<Foam::mapDistribute> Foam::advancingFrontAMI::calcProcMap
}
// Convert dynamicList to labelList
sendMap.setSize(Pstream::nProcs(comm_));
sendMap.setSize(Pstream::nProcs(comm()));
forAll(sendMap, proci)
{
sendMap[proci].transfer(dynSendMap[proci]);
@ -344,7 +351,7 @@ Foam::autoPtr<Foam::mapDistribute> Foam::advancingFrontAMI::calcProcMap
std::move(sendMap),
false, //subHasFlip
false, //constructHasFlip
comm_
comm()
);
}

16
src/meshTools/AMIInterpolation/AMIInterpolation/faceAreaWeightAMI/faceAreaWeightAMI.C
View File

@ -711,9 +711,9 @@ bool Foam::faceAreaWeightAMI::calculate
tgtWeights_[i].transfer(tgtWght[i]);
}
if (distributed())
if (distributed() && comm() != -1)
{
const label myRank = UPstream::myProcNo(comm_);
const label myRank = UPstream::myProcNo(comm());
// Allocate unique tag for all comms
const int oldTag = UPstream::incrMsgType();
@ -721,8 +721,8 @@ bool Foam::faceAreaWeightAMI::calculate
const primitivePatch& tgtPatch0 = this->tgtPatch0();
// Create global indexing for each original patch
globalIndex globalSrcFaces(srcPatch0.size(), comm_);
globalIndex globalTgtFaces(tgtPatch0.size(), comm_);
globalIndex globalSrcFaces(srcPatch0.size(), comm());
globalIndex globalTgtFaces(tgtPatch0.size(), comm());
for (labelList& addressing : srcAddress_)
{
@ -754,7 +754,7 @@ bool Foam::faceAreaWeightAMI::calculate
ListOps::appendEqOp<label>(),
flipOp(), // flip operation
UPstream::msgType()+77431,
comm_
comm()
);
mapDistributeBase::distribute
@ -771,7 +771,7 @@ bool Foam::faceAreaWeightAMI::calculate
ListOps::appendEqOp<scalar>(),
flipOp(),
UPstream::msgType()+77432,
comm_
comm()
);
// Note: using patch face areas calculated by the AMI method
@ -787,7 +787,7 @@ bool Foam::faceAreaWeightAMI::calculate
tgtAddress_,
cMapSrc,
UPstream::msgType()+77433,
comm_
comm()
)
);
@ -800,7 +800,7 @@ bool Foam::faceAreaWeightAMI::calculate
srcAddress_,
cMapTgt,
UPstream::msgType()+77434,
comm_
comm()
)
);

16
src/meshTools/AMIInterpolation/AMIInterpolation/faceAreaWeightAMI2D/faceAreaWeightAMI2D.C
View File

@ -425,9 +425,9 @@ bool Foam::faceAreaWeightAMI2D::calculate
tgtWeights_[i].transfer(tgtWght[i]);
}
if (distributed())
if (distributed() && comm() != -1)
{
const label myRank = UPstream::myProcNo(comm_);
const label myRank = UPstream::myProcNo(comm());
// Allocate unique tag for all comms
const int oldTag = UPstream::incrMsgType();
@ -435,8 +435,8 @@ bool Foam::faceAreaWeightAMI2D::calculate
const primitivePatch& tgtPatch0 = this->tgtPatch0();
// Create global indexing for each original patch
const globalIndex globalSrcFaces(srcPatch0.size(), comm_);
const globalIndex globalTgtFaces(tgtPatch0.size(), comm_);
const globalIndex globalSrcFaces(srcPatch0.size(), comm());
const globalIndex globalTgtFaces(tgtPatch0.size(), comm());
for (labelList& addressing : srcAddress_)
{
@ -468,7 +468,7 @@ bool Foam::faceAreaWeightAMI2D::calculate
ListOps::appendEqOp<label>(),
flipOp(), // flip operation
UPstream::msgType()+77431,
comm_
comm()
);
mapDistributeBase::distribute
@ -485,7 +485,7 @@ bool Foam::faceAreaWeightAMI2D::calculate
ListOps::appendEqOp<scalar>(),
flipOp(), // flip operation
UPstream::msgType()+77432,
comm_
comm()
);
// Note: using patch face areas calculated by the AMI method
@ -501,7 +501,7 @@ bool Foam::faceAreaWeightAMI2D::calculate
tgtAddress_,
cMapSrc,
UPstream::msgType()+77433,
comm_
comm()
)
);
@ -514,7 +514,7 @@ bool Foam::faceAreaWeightAMI2D::calculate
srcAddress_,
cMapTgt,
UPstream::msgType()+77434,
comm_
comm()
)
);

34
src/meshTools/AMIInterpolation/AMIInterpolation/nearestFaceAMI/nearestFaceAMI.C
View File

@ -5,7 +5,7 @@
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2020-2022 OpenCFD Ltd.
Copyright (C) 2020-2024 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -47,15 +47,15 @@ Foam::autoPtr<Foam::mapDistribute> Foam::nearestFaceAMI::calcFaceMap
) const
{
// Generate the list of processor bounding boxes for tgtPatch
List<boundBox> procBbs(Pstream::nProcs());
procBbs[Pstream::myProcNo()] =
boundBox(tgtPatch.points(), tgtPatch.meshPoints(), true);
Pstream::allGatherList(procBbs);
List<boundBox> procBbs(Pstream::nProcs(comm()));
procBbs[Pstream::myProcNo(comm())] =
boundBox(tgtPatch.points(), tgtPatch.meshPoints(), false);
Pstream::allGatherList(procBbs, UPstream::msgType(), comm());
// Identify which of my local src faces intersect with each processor
// tgtPatch bb within the current match's search distance
const pointField& srcCcs = srcPatch.faceCentres();
List<DynamicList<label>> dynSendMap(Pstream::nProcs());
List<DynamicList<label>> dynSendMap(Pstream::nProcs(comm()));
forAll(localInfo, srcFacei)
{
@ -66,7 +66,7 @@ Foam::autoPtr<Foam::mapDistribute> Foam::nearestFaceAMI::calcFaceMap
forAll(procBbs, proci)
{
if (proci != Pstream::myProcNo())
if (proci != Pstream::myProcNo(comm()))
{
if (procBbs[proci].overlaps(srcCcs[srcFacei], r2))
{
@ -77,7 +77,7 @@ Foam::autoPtr<Foam::mapDistribute> Foam::nearestFaceAMI::calcFaceMap
}
// Convert dynamicList to labelList
labelListList sendMap(Pstream::nProcs());
labelListList sendMap(Pstream::nProcs(comm()));
forAll(sendMap, proci)
{
sendMap[proci].transfer(dynSendMap[proci]);
@ -89,7 +89,13 @@ Foam::autoPtr<Foam::mapDistribute> Foam::nearestFaceAMI::calcFaceMap
}
}
return autoPtr<mapDistribute>::New(std::move(sendMap));
return autoPtr<mapDistribute>::New
(
std::move(sendMap),
false,
false,
comm()
);
}
@ -108,10 +114,10 @@ Foam::autoPtr<Foam::mapDistribute> Foam::nearestFaceAMI::calcDistributed
}
// Create global indexing for tgtPatch
globalIndex globalTgtCells(tgt.size());
globalIndex globalTgtCells(tgt.size(), comm());
const label myRank = UPstream::myProcNo(comm_);
const label myRank = UPstream::myProcNo(comm());
// First pass
@ -205,7 +211,7 @@ Foam::autoPtr<Foam::mapDistribute> Foam::nearestFaceAMI::calcDistributed
nearestEqOp(),
identityOp(), // No flipping
UPstream::msgType(),
comm_
comm()
);
@ -234,7 +240,7 @@ Foam::autoPtr<Foam::mapDistribute> Foam::nearestFaceAMI::calcDistributed
srcToTgtAddr,
cMap,
UPstream::msgType(),
comm_
comm()
);
}
@ -297,7 +303,7 @@ bool Foam::nearestFaceAMI::calculate
// TODO: implement symmetric calculation controls; assume yes for now
bool symmetric_ = true;
if (this->distributed())
if (this->distributed() && comm() != -1)
{
tgtMapPtr_ =
calcDistributed

4
src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicACMIGAMGInterfaceField/cyclicACMIGAMGInterfaceField.C
View File

@ -184,7 +184,7 @@ void Foam::cyclicACMIGAMGInterfaceField::initInterfaceMatrixUpdate
: cyclicACMIInterface_.neighbPatch().AMI()
);
if (AMI.distributed())
if (AMI.distributed() && AMI.comm() != -1)
{
DebugPout<< "cyclicACMIFvPatchField::initInterfaceMatrixUpdate() :"
<< " interface:" << cyclicACMIInterface_.index()
@ -277,7 +277,7 @@ void Foam::cyclicACMIGAMGInterfaceField::updateInterfaceMatrix
<< endl;
if (AMI.distributed())
if (AMI.distributed() && AMI.comm() != -1)
{
const auto& map =
(

4
src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicAMIGAMGInterfaceField/cyclicAMIGAMGInterfaceField.C
View File

@ -184,7 +184,7 @@ void Foam::cyclicAMIGAMGInterfaceField::initInterfaceMatrixUpdate
: cyclicAMIInterface_.neighbPatch().AMI()
);
if (AMI.distributed())
if (AMI.distributed() && AMI.comm() != -1)
{
//DebugPout<< "cyclicAMIFvPatchField::initInterfaceMatrixUpdate() :"
// << " interface:" << cyclicAMIInterface_.index()
@ -284,7 +284,7 @@ void Foam::cyclicAMIGAMGInterfaceField::updateInterfaceMatrix
// << " AMI low-weight:" << AMI.applyLowWeightCorrection()
// << endl;
if (AMI.distributed())
if (AMI.distributed() && AMI.comm() != -1)
{
if (commsType != UPstream::commsTypes::nonBlocking)
{

983
src/meshTools/AMIInterpolation/GAMG/interfaces/cyclicACMIGAMGInterface/cyclicACMIGAMGInterface.C
View File

File diff suppressed because it is too large Load Diff

15
src/meshTools/AMIInterpolation/GAMG/interfaces/cyclicACMIGAMGInterface/cyclicACMIGAMGInterface.H
View File

@ -35,8 +35,8 @@ SourceFiles
\*---------------------------------------------------------------------------*/
#ifndef cyclicACMIGAMGInterface_H
#define cyclicACMIGAMGInterface_H
#ifndef Foam_cyclicACMIGAMGInterface_H
#define Foam_cyclicACMIGAMGInterface_H
#include "GAMGInterface.H"
#include "cyclicACMILduInterface.H"
@ -68,6 +68,11 @@ class cyclicACMIGAMGInterface
//- AMI interface
autoPtr<AMIPatchToPatchInterpolation> amiPtr_;
//- rank in AMI with respect to parent communicator.
//- Used for combining contributions from different processors if the
//- destination processor was not in the original set.
label myProcNo_;
// Private Member Functions
@ -213,6 +218,12 @@ public:
return reverseT_;
}
//- -1 or old local rank
virtual label myProcNo() const
{
return myProcNo_;
}
// I/O

1179
src/meshTools/AMIInterpolation/GAMG/interfaces/cyclicAMIGAMGInterface/cyclicAMIGAMGInterface.C
View File

File diff suppressed because it is too large Load Diff

11
src/meshTools/AMIInterpolation/GAMG/interfaces/cyclicAMIGAMGInterface/cyclicAMIGAMGInterface.H
View File

@ -68,6 +68,11 @@ class cyclicAMIGAMGInterface
//- AMI interface
autoPtr<AMIPatchToPatchInterpolation> amiPtr_;
//- rank in AMI with respect to parent communicator.
//- Used for combining contributions from different processors if the
//- destination processor was not in the original set.
label myProcNo_;
// Private Member Functions
@ -213,6 +218,12 @@ public:
return reverseT_;
}
//- -1 or old local rank
virtual label myProcNo() const
{
return myProcNo_;
}
// I/O

16
src/meshTools/AMIInterpolation/patches/cyclicAMI/cyclicAMIPolyPatch/cyclicAMIPolyPatchTemplates.C
View File

@ -183,7 +183,7 @@ void Foam::cyclicAMIPolyPatch::initInterpolateUntransformed
{
const auto& AMI = (owner() ? this->AMI() : neighbPatch().AMI());
if (AMI.distributed())
if (AMI.distributed() && AMI.comm() != -1)
{
const auto& map = (owner() ? AMI.tgtMap() : AMI.srcMap());
@ -213,7 +213,7 @@ void Foam::cyclicAMIPolyPatch::initInterpolate
{
const auto& AMI = (owner() ? this->AMI() : neighbPatch().AMI());
if (!AMI.distributed())
if (!AMI.distributed() || AMI.comm() == -1)
{
return;
}
@ -273,12 +273,20 @@ Foam::tmp<Foam::Field<Type>> Foam::cyclicAMIPolyPatch::interpolate
const UList<Type>& defaultValues
) const
{
auto tresult = tmp<Field<Type>>::New(this->size(), Zero);
const auto& AMI = (owner() ? this->AMI() : neighbPatch().AMI());
const auto& map = (owner() ? AMI.tgtMap() : AMI.srcMap());
Field<Type> work;
if (AMI.distributed())
{
if (AMI.comm() == -1)
{
return tresult;
}
const auto& map = (owner() ? AMI.tgtMap() : AMI.srcMap());
// Receive (= copy) data from buffers into work. TBD: receive directly
// into slices of work.
map.receive
@ -291,8 +299,6 @@ Foam::tmp<Foam::Field<Type>> Foam::cyclicAMIPolyPatch::interpolate
}
const Field<Type>& fld = (AMI.distributed() ? work : localFld);
auto tresult = tmp<Field<Type>>::New(this->size(), Zero);
// Rotate fields (vector and non-spherical tensors)
constexpr bool transform_supported = is_rotational_vectorspace_v<Type>;

2
src/meshTools/AMIInterpolation/patches/cyclicAMI/cyclicAMIPolyPatch/cyclicAMIPolyPatchTopologyChange.C
View File

@ -312,7 +312,7 @@ void Foam::cyclicAMIPolyPatch::setAMIFaces()
autoPtr<mapDistribute> srcToTgtMap1;
autoPtr<mapDistribute> tgtToSrcMap1;
if (AMIPtr_->distributed())
if (AMIPtr_->distributed() && AMIPtr_().comm() != -1)
{
// Parallel running

4
src/sampling/meshToMesh/meshToMeshTemplates.C
View File

@ -537,7 +537,7 @@ void Foam::meshToMesh::mapSrcToTgt
(
AMIList[i].singlePatchProc(),
(
AMIList[i].distributed()
(AMIList[i].distributed() && AMIList[i].comm() != -1)
? AMIList[i].hasSrcMap() // pointer to map
: nullptr
),
@ -769,7 +769,7 @@ void Foam::meshToMesh::mapTgtToSrc
(
AMIList[i].singlePatchProc(),
(
AMIList[i].distributed()
(AMIList[i].distributed() && AMIList[i].comm() != -1)
? AMIList[i].hasTgtMap() // pointer to map
: nullptr
),