zhyang-dev/openfoam
1
0
Fork 0
mirror of https://develop.openfoam.com/Development/openfoam.git synced 2025-11-28 03:28:01 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

ENH: AMIInterpolation: use local communicator

Browse Source 
- allocate communicator with only those ranks that have
patch faces
- use this communicator everywhere
- communicator preserved during mesh motion
This commit is contained in:
mattijs
2024-12-05 16:34:54 +00:00
parent f276016896
commit 34c933c1ce
25 changed files with 1884 additions and 1079 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
etc/controlDict
View File

@ -240,6 +240,12 @@ OptimisationSwitches
// from all the individual wall patches. Set to 0/false to revert to
// <v2412 behaviour
//useCombinedWallPatch 0;
//- Use local communicator for AMI communication. Default for v2506.
//localAMIComm = 0; // old behaviour : all ranks included
//localAMIComm = 1; // (default) only ranks that have patch faces
//localAMIComm = 2; // like 1 but always include rank 0 so messages
// still come from processor0
}

4
src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C
View File

@ -328,7 +328,7 @@ void Foam::GAMGAgglomeration::agglomerateLduAddressing
tmp<labelField> restrictMapInternalField;
// The finest mesh uses patchAddr from the original lduAdressing.
// the coarser levels create thei own adressing for faceCells
// the coarser levels create their own adressing for faceCells
if (fineLevelIndex == 0)
{
restrictMapInternalField =
@ -466,7 +466,7 @@ void Foam::GAMGAgglomeration::procAgglomerateLduAddressing
procBoundaryFaceMap_.set(levelIndex, new labelListListList(0));
// Collect meshes
// Collect meshes. Does no renumbering
PtrList<lduPrimitiveMesh> otherMeshes;
lduPrimitiveMesh::gather(comm, myMesh, otherMeshes);

4
src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/masterCoarsestGAMGProcAgglomeration/masterCoarsestGAMGProcAgglomeration.C
View File

@ -182,8 +182,8 @@ bool Foam::masterCoarsestGAMGProcAgglomeration::agglomerate()
for (const auto& p : masterToProcs)
{
Info<< '\t' << p[0]
<< '\t' << p.size()
<< '\t'
<< "\t\t" << p.size()
<< "\t\t"
<< flatOutput(SubList<label>(p, p.size()-1, 1))
<< endl;
}

266
src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeBase.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2015-2017 OpenFOAM Foundation
Copyright (C) 2015-2024 OpenCFD Ltd.
Copyright (C) 2015-2025 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -1001,17 +1001,39 @@ Foam::mapDistributeBase::mapDistributeBase
constructSize_(0),
subHasFlip_(false),
constructHasFlip_(false),
comm_(-1),
comm_(newComm),
schedulePtr_(nullptr)
{
if (maps.empty())
// localRanks : gives for every map the index (=rank in original
// communicator) in subMap,constructMap that refers to
// the local data. -1 if the map did not contain any data
// (can happen for e.g. agglomerating of cyclicAMI data)
// newToOldRanks : gives for every rank in the newComm the ranks in the
// maps that agglomerate to it. This is used to decide
// - data was local and stays local
// - data that was remote and now becomes local
// - and same for remote
// Currently can contain -1 entries. Probably should be
// filtered...
// Find set index
label validMapi = -1;
forAll(maps, mapi)
{
if (maps.set(mapi) && localRanks[mapi] != -1)
{
validMapi = mapi;
break;
}
}
if (validMapi == -1)
{
return;
}
comm_ = newComm;
subHasFlip_ = maps[0].subHasFlip();
constructHasFlip_ = maps[0].constructHasFlip();
subHasFlip_ = maps[validMapi].subHasFlip();
constructHasFlip_ = maps[validMapi].constructHasFlip();
const label nNewRanks = newToOldRanks.size();
const label myNewRank = UPstream::myProcNo(newComm);
@ -1031,15 +1053,20 @@ Foam::mapDistributeBase::mapDistributeBase
}
// Sanity checks
const auto& map0 = maps[0];
const auto& map0 = maps[validMapi];
forAll(maps, mapi)
{
if (!maps.set(mapi) || localRanks[mapi] == -1)
{
continue;
}
const auto& map = maps[mapi];
if
(
(map.comm() != map0.comm())
|| (map.subHasFlip() != map0.subHasFlip())
//(map.comm() != map0.comm())
(map.subHasFlip() != map0.subHasFlip())
|| (map.constructHasFlip() != map0.constructHasFlip())
)
{
@ -1050,6 +1077,9 @@ Foam::mapDistributeBase::mapDistributeBase
<< " subHasFlip:" << map.subHasFlip()
<< " constructHasFlip:" << map.constructHasFlip()
<< " which is different from map 0 "
<< " comm:" << map0.comm()
<< " subHasFlip:" << map0.subHasFlip()
<< " constructHasFlip:" << map0.constructHasFlip()
<< exit(FatalError);
}
@ -1069,6 +1099,38 @@ Foam::mapDistributeBase::mapDistributeBase
}
// Determine start of constructed remote data. This is used to get the
// local offset which can then be used to get the relative subMap location.
labelListList startOfRemote(maps.size());
forAll(maps, mapi)
{
if (!maps.set(mapi))
{
continue;
}
const label nOldRanks = maps[mapi].constructMap().size();
labelList& starts = startOfRemote[mapi];
starts.setSize(nOldRanks, labelMax);
forAll(maps[mapi].constructMap(), oldRanki)
{
const labelList& map = maps[mapi].constructMap()[oldRanki];
forAll(map, i)
{
const label index
(
constructHasFlip_
? mag(map[i])-1
: map[i]
);
starts[oldRanki] = min(starts[oldRanki], index);
}
}
}
constructMap_.resize_nocopy(nNewRanks);
subMap_.resize_nocopy(nNewRanks);
@ -1081,7 +1143,14 @@ Foam::mapDistributeBase::mapDistributeBase
forAll(maps, mapi)
{
startOfLocal[mapi] = constructi;
const label localRank = localRanks[mapi];
if (!maps.set(mapi) || localRank == -1)
{
continue;
}
const auto& map = maps[mapi].constructMap()[localRank];
// Presize compaction array
@ -1093,31 +1162,6 @@ Foam::mapDistributeBase::mapDistributeBase
startOfLocal.last() = constructi;
// Determine start of constructed remote data. This is used to get the
// local offset which can then be used to get the relative subMap location.
labelListList startOfRemote(maps.size());
forAll(maps, mapi)
{
const label nOldProcs = maps[mapi].constructMap().size();
labelList& starts = startOfRemote[mapi];
starts.setSize(nOldProcs, labelMax);
forAll(maps[mapi].constructMap(), oldProci)
{
const labelList& map = maps[mapi].constructMap()[oldProci];
forAll(map, i)
{
const label index
(
constructHasFlip_
? mag(map[i])-1
: map[i]
);
starts[oldProci] = min(starts[oldProci], index);
}
}
}
// Construct map
// ~~~~~~~~~~~~~
@ -1141,6 +1185,12 @@ Foam::mapDistributeBase::mapDistributeBase
forAll(maps, mapi)
{
const label localRank = localRanks[mapi];
if (!maps.set(mapi) || localRank == -1)
{
continue;
}
const auto& map = maps[mapi].constructMap()[localRank];
const label offset = startOfLocal[mapi];
@ -1166,22 +1216,68 @@ Foam::mapDistributeBase::mapDistributeBase
}
}
}
if (constructi != startOfLocal.last())
{
FatalErrorInFunction
<< "constructi:" << constructi
<< " startOfLocal:" << startOfLocal.last()
<< exit(FatalError);
}
}
// Filter remote construct data
{
// Remote ranks that are now local
// - store new index for mapping stencils
// - no need to construct since already
const auto& oldProcs = newToOldRanks[myNewRank];
const auto& oldRanks = newToOldRanks[myNewRank];
// Determine number of old ranks. Note: should have passed in old-to-new
// ranks instead of new-to-old ranks?
label maxOldRank = -1;
for (const auto& elems : newToOldRanks)
{
for (const label el : elems)
{
maxOldRank = max(maxOldRank, el);
}
}
// Construct mapping from oldRanks to map. Note: could probably
// use some ListOps invert routine ...
labelList oldRankToMap(maxOldRank+1, -1);
{
forAll(localRanks, mapi)
{
const label localRank = localRanks[mapi];
if (localRank != -1)
{
oldRankToMap[localRank] = mapi;
}
}
}
forAll(maps, mapi)
{
for (const label oldProci : oldProcs)
if (!maps.set(mapi) || localRanks[mapi] == -1)
{
if (oldProci != localRanks[mapi])
continue;
}
for (const label oldRanki : oldRanks)
{
const auto& map = maps[mapi].constructMap()[oldProci];
if (oldRanki == -1)
{
continue;
}
if (oldRanki != localRanks[mapi])
{
// Note:
// - constructMap is sized with the map's communicator
const auto& map = maps[mapi].constructMap()[oldRanki];
if (!map.size())
{
@ -1191,11 +1287,11 @@ Foam::mapDistributeBase::mapDistributeBase
// The slots come from a local map so we can look up the
// new location
const label sourceMapi = localRanks.find(oldProci);
const label sourceMapi = oldRankToMap[oldRanki];
const auto& subMap =
maps[sourceMapi].subMap()[localRanks[mapi]];
//Pout<< "From oldRank:" << oldProci
//Pout<< "From oldRank:" << oldRanki
// << " sending to masterRank:" << localRanks[mapi]
// << " elements:" << flatOutput(subMap)
// << nl
@ -1205,7 +1301,7 @@ Foam::mapDistributeBase::mapDistributeBase
if (map.size() != subMap.size())
{
FatalErrorInFunction << "Problem:"
<< "oldProci:" << oldProci
<< "oldRanki:" << oldRanki
<< " mapi:" << mapi
<< " constructMap:" << map.size()
<< " sourceMapi:" << sourceMapi
@ -1215,7 +1311,7 @@ Foam::mapDistributeBase::mapDistributeBase
const label offset = startOfLocal[sourceMapi];
// Construct map starts after the local data
const label nMapLocal = startOfRemote[mapi][oldProci];
const label nMapLocal = startOfRemote[mapi][oldRanki];
auto& cptMap = compactMaps[mapi];
forAll(map, i)
@ -1238,7 +1334,7 @@ Foam::mapDistributeBase::mapDistributeBase
)
{
FatalErrorInFunction<< "Duplicate insertion"
<< "From oldProc:" << oldProci
<< "From oldRank:" << oldRanki
<< " on map:" << mapi
<< " at index:" << i
<< " have construct slot:" << index
@ -1260,32 +1356,51 @@ Foam::mapDistributeBase::mapDistributeBase
// Either loop over all old ranks and filter out ones already handled
// or loop over all new ranks and avoid myNewRank
forAll(newToOldRanks, newProci)
forAll(newToOldRanks, newRanki)
{
if (newProci != myNewRank)
if (newRanki != myNewRank)
{
const auto& oldProcs = newToOldRanks[newProci];
const auto& oldRanks = newToOldRanks[newRanki];
label allSize = 0;
forAll(maps, mapi)
{
for (const label oldProci : oldProcs)
if (!maps.set(mapi))
{
allSize += maps[mapi].constructMap()[oldProci].size();
continue;
}
for (const label oldRanki : oldRanks)
{
if (oldRanki == -1)
{
continue;
}
allSize += maps[mapi].constructMap()[oldRanki].size();
}
}
labelList& myConstruct = constructMap_[newProci];
labelList& myConstruct = constructMap_[newRanki];
myConstruct.resize_nocopy(allSize);
allSize = 0;
forAll(maps, mapi)
{
for (const label oldProci : oldProcs)
if (!maps.set(mapi))
{
const auto& map = maps[mapi].constructMap()[oldProci];
continue;
}
for (const label oldRanki : oldRanks)
{
if (oldRanki == -1)
{
continue;
}
const auto& map = maps[mapi].constructMap()[oldRanki];
// Construct map starts after the local data
const label nMapLocal = startOfRemote[mapi][oldProci];
const label nMapLocal = startOfRemote[mapi][oldRanki];
SubList<label> slice(myConstruct, map.size(), allSize);
if (constructHasFlip_)
@ -1340,6 +1455,12 @@ Foam::mapDistributeBase::mapDistributeBase
forAll(maps, mapi)
{
const label localRank = localRanks[mapi];
if (!maps.set(mapi) || localRank == -1)
{
continue;
}
allSize += maps[mapi].subMap()[localRank].size();
}
@ -1349,6 +1470,12 @@ Foam::mapDistributeBase::mapDistributeBase
forAll(maps, mapi)
{
const label localRank = localRanks[mapi];
if (!maps.set(mapi) || localRank == -1)
{
continue;
}
const auto& map = maps[mapi].subMap()[localRank];
SubList<label> slice(mySub, map.size(), allSize);
@ -1377,30 +1504,49 @@ Foam::mapDistributeBase::mapDistributeBase
}
}
// Filter remote sub data
forAll(newToOldRanks, newProci)
forAll(newToOldRanks, newRanki)
{
if (newProci != myNewRank)
if (newRanki != myNewRank)
{
const auto& oldProcs = newToOldRanks[newProci];
const auto& oldRanks = newToOldRanks[newRanki];
label allSize = 0;
forAll(maps, mapi)
{
for (const label oldProci : oldProcs)
if (!maps.set(mapi))
{
allSize += maps[mapi].subMap()[oldProci].size();
continue;
}
for (const label oldRanki : oldRanks)
{
if (oldRanki == -1)
{
continue;
}
allSize += maps[mapi].subMap()[oldRanki].size();
}
}
labelList& mySub = subMap_[newProci];
labelList& mySub = subMap_[newRanki];
mySub.resize_nocopy(allSize);
allSize = 0;
for (const label oldProci : oldProcs)
for (const label oldRanki : oldRanks)
{
if (oldRanki == -1)
{
continue;
}
forAll(maps, mapi)
{
const auto& map = maps[mapi].subMap()[oldProci];
if (!maps.set(mapi))
{
continue;
}
const auto& map = maps[mapi].subMap()[oldRanki];
SubList<label> slice(mySub, map.size(), allSize);
if (subHasFlip_)
{

22
src/finiteVolume/fields/fvPatchFields/constraint/cyclicACMI/cyclicACMIFvPatchField.C
View File

@ -345,7 +345,9 @@ template<class Type>
Foam::tmp<Foam::Field<Type>>
Foam::cyclicACMIFvPatchField<Type>::patchNeighbourField() const
{
if (this->ownerAMI().distributed() && cacheNeighbourField())
const auto& AMI = this->ownerAMI();
if (AMI.distributed() && cacheNeighbourField() && AMI.comm() != -1)
{
if (!this->ready())
{
@ -441,7 +443,9 @@ void Foam::cyclicACMIFvPatchField<Type>::initEvaluate
this->updateCoeffs();
}
if (this->ownerAMI().distributed() && cacheNeighbourField())
const auto& AMI = this->ownerAMI();
if (AMI.distributed() && cacheNeighbourField() && AMI.comm() != -1)
{
if (commsType != UPstream::commsTypes::nonBlocking)
{
@ -502,7 +506,7 @@ void Foam::cyclicACMIFvPatchField<Type>::evaluate
const auto& AMI = this->ownerAMI();
if (AMI.distributed() && cacheNeighbourField())
if (AMI.distributed() && cacheNeighbourField() && AMI.comm() != -1)
{
// Calculate patchNeighbourField
if (commsType != UPstream::commsTypes::nonBlocking)
@ -564,7 +568,9 @@ void Foam::cyclicACMIFvPatchField<Type>::initInterfaceMatrixUpdate
const Pstream::commsTypes commsType
) const
{
if (this->ownerAMI().distributed())
const auto& AMI = this->ownerAMI();
if (AMI.distributed() && AMI.comm() != -1)
{
// Start sending
if (commsType != UPstream::commsTypes::nonBlocking)
@ -640,7 +646,9 @@ void Foam::cyclicACMIFvPatchField<Type>::updateInterfaceMatrix
solveScalarField pnf;
if (this->ownerAMI().distributed())
const auto& AMI = this->ownerAMI();
if (AMI.distributed() && AMI.comm() != -1)
{
if (commsType != UPstream::commsTypes::nonBlocking)
{
@ -700,7 +708,7 @@ void Foam::cyclicACMIFvPatchField<Type>::initInterfaceMatrixUpdate
{
const auto& AMI = this->ownerAMI();
if (AMI.distributed())
if (AMI.distributed() && AMI.comm() != -1)
{
if (commsType != UPstream::commsTypes::nonBlocking)
{
@ -764,7 +772,7 @@ void Foam::cyclicACMIFvPatchField<Type>::updateInterfaceMatrix
Field<Type> pnf;
if (AMI.distributed())
if (AMI.distributed() && AMI.comm() != -1)
{
if (commsType != UPstream::commsTypes::nonBlocking)
{

20
src/finiteVolume/fields/fvPatchFields/constraint/cyclicAMI/cyclicAMIFvPatchField.C
View File

@ -338,7 +338,9 @@ template<class Type>
Foam::tmp<Foam::Field<Type>>
Foam::cyclicAMIFvPatchField<Type>::patchNeighbourField() const
{
if (this->ownerAMI().distributed() && cacheNeighbourField())
const auto& AMI = this->ownerAMI();
if (AMI.distributed() && cacheNeighbourField() && AMI.comm() != -1)
{
if (!this->ready())
{
@ -442,7 +444,9 @@ void Foam::cyclicAMIFvPatchField<Type>::initEvaluate
this->updateCoeffs();
}
if (this->ownerAMI().distributed() && cacheNeighbourField())
const auto& AMI = this->ownerAMI();
if (AMI.distributed() && cacheNeighbourField() && AMI.comm() != -1)
{
//DebugPout
// << "*** cyclicAMIFvPatchField::initEvaluate() :"
@ -506,7 +510,7 @@ void Foam::cyclicAMIFvPatchField<Type>::evaluate
const auto& AMI = this->ownerAMI();
if (AMI.distributed() && cacheNeighbourField())
if (AMI.distributed() && cacheNeighbourField() && AMI.comm() != -1)
{
// Calculate patchNeighbourField
if (commsType != UPstream::commsTypes::nonBlocking)
@ -574,7 +578,9 @@ void Foam::cyclicAMIFvPatchField<Type>::initInterfaceMatrixUpdate
const Pstream::commsTypes commsType
) const
{
if (this->ownerAMI().distributed())
const auto& AMI = this->ownerAMI();
if (AMI.distributed() && AMI.comm() != -1)
{
// Start sending
if (commsType != UPstream::commsTypes::nonBlocking)
@ -645,7 +651,7 @@ void Foam::cyclicAMIFvPatchField<Type>::updateInterfaceMatrix
solveScalarField pnf;
if (AMI.distributed())
if (AMI.distributed() && AMI.comm() != -1)
{
if (commsType != UPstream::commsTypes::nonBlocking)
{
@ -714,7 +720,7 @@ void Foam::cyclicAMIFvPatchField<Type>::initInterfaceMatrixUpdate
{
const auto& AMI = this->ownerAMI();
if (AMI.distributed())
if (AMI.distributed() && AMI.comm() != -1)
{
if (commsType != UPstream::commsTypes::nonBlocking)
{
@ -783,7 +789,7 @@ void Foam::cyclicAMIFvPatchField<Type>::updateInterfaceMatrix
Field<Type> pnf;
if (AMI.distributed())
if (AMI.distributed() && AMI.comm() != -1)
{
if (commsType != UPstream::commsTypes::nonBlocking)
{

2
src/finiteVolume/fvMesh/fvPatches/constraint/cyclicAMI/cyclicAMIFvPatch.C
View File

@ -289,7 +289,7 @@ void Foam::cyclicAMIFvPatch::movePoints()
}
scalarField srcMeshPhi(phip);
if (AMI().distributed())
if (AMI().distributed() && AMI().comm() != -1)
{
AMI().srcMap().distribute(srcMeshPhi);
}

186
src/meshTools/AMIInterpolation/AMIInterpolation/AMIInterpolation.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2015-2023 OpenCFD Ltd.
Copyright (C) 2015-2024 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -33,6 +33,7 @@ License
#include "profiling.H"
#include "triangle.H"
#include "OFstream.H"
#include "registerSwitch.H"
#include <numeric> // For std::iota
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
@ -46,6 +47,17 @@ namespace Foam
bool Foam::AMIInterpolation::cacheIntersections_ = false;
int Foam::AMIInterpolation::localComm_
(
debug::optimisationSwitch("localAMIComm", 1)
);
registerOptSwitch
(
"localAMIComm",
int,
Foam::AMIInterpolation::localComm_
);
// * * * * * * * * * * * * Protected Member Functions * * * * * * * * * * * //
@ -77,7 +89,9 @@ Foam::AMIInterpolation::createTree
Foam::label Foam::AMIInterpolation::calcDistribution
(
const primitivePatch& srcPatch,
const primitivePatch& tgtPatch
const primitivePatch& tgtPatch,
const label comm,
autoPtr<UPstream::communicator>& geomComm
) const
{
// Either not parallel or no faces on any processor
@ -85,12 +99,40 @@ Foam::label Foam::AMIInterpolation::calcDistribution
if (Pstream::parRun())
{
bool hasLocalFaces = false;
if (localComm_ == 0)
{
// Backwards compatible : all processors involved
hasLocalFaces = true;
}
else if (localComm_ == 1)
{
// Only if locally have faces
hasLocalFaces = (srcPatch.size() > 0 || tgtPatch.size() > 0);
}
else
{
// If master (so messages always come from master) or if locally
// have faces
hasLocalFaces =
(
UPstream::master(comm)
|| srcPatch.size() > 0
|| tgtPatch.size() > 0
);
}
// Better to use MPI_Comm_split? So only provide local information
// (hasLocalFaces). Problem is that we don't know who else is in the
// set. Whereas now, because we all loop over the same data in the same
// order we coud find out (except we don't use this information?)
const bitSet hasFaces
(
UPstream::listGatherValues<bool>
UPstream::allGatherValues<bool>
(
(srcPatch.size() > 0 || tgtPatch.size() > 0),
comm_
hasLocalFaces,
comm
)
);
@ -98,18 +140,48 @@ Foam::label Foam::AMIInterpolation::calcDistribution
if (nHaveFaces == 1)
{
// Release any previously allocated communicator
geomComm.clear();
proci = hasFaces.find_first();
DebugInFunction
<< "AMI local to processor" << proci << endl;
}
else if (nHaveFaces > 1)
{
proci = -1;
DebugInFunction
<< "AMI split across multiple processors" << endl;
if (hasLocalFaces)
{
geomComm.reset
(
new UPstream::communicator
(
comm,
hasFaces.sortedToc()
)
);
if (debug)
{
Pout<< "Allocated geomComm:" << geomComm()
<< " from " << nHaveFaces
<< " processors out of " << UPstream::nProcs(comm)
<< endl;
}
}
else
{
geomComm.reset(new UPstream::communicator());
if (debug & 2)
{
Pout<< "Allocated dummy geomComm:" << geomComm()
<< " since no src:" << srcPatch.size()
<< " and no tgt:" << tgtPatch.size() << endl;
}
}
Pstream::broadcast(proci, comm_);
proci = -1;
DebugInFunction
<< "AMI split across multiple processors "
<< flatOutput(hasFaces.sortedToc()) << endl;
}
}
return proci;
@ -207,7 +279,7 @@ void Foam::AMIInterpolation::normaliseWeights
}
}
if (output)
if (output && comm != -1)
{
// Note: change global communicator since gMin,gAverage etc don't
// support user communicator
@ -216,7 +288,8 @@ void Foam::AMIInterpolation::normaliseWeights
if (returnReduceOr(wght.size()))
{
Info<< indent
Info.masterStream(comm)
<< indent
<< "AMI: Patch " << patchName
<< " sum(weights)"
<< " min:" << gMin(wghtSum)
@ -227,7 +300,8 @@ void Foam::AMIInterpolation::normaliseWeights
if (nLow)
{
Info<< indent
Info.masterStream(comm)
<< indent
<< "AMI: Patch " << patchName
<< " identified " << nLow
<< " faces with weights less than " << lowWeightTol
@ -260,14 +334,14 @@ void Foam::AMIInterpolation::agglomerate
{
addProfiling(ami, "AMIInterpolation::agglomerate");
label sourceCoarseSize =
const label sourceCoarseSize =
(
sourceRestrictAddressing.size()
? max(sourceRestrictAddressing)+1
: 0
);
label targetCoarseSize =
const label targetCoarseSize =
(
targetRestrictAddressing.size()
? max(targetRestrictAddressing)+1
@ -289,8 +363,22 @@ void Foam::AMIInterpolation::agglomerate
// Agglomerate weights and indices
if (targetMapPtr)
{
// We are involved in the communicator but our maps are still empty.
// Fix 'm up so they are the same size as the communicator.
const mapDistribute& map = *targetMapPtr;
if (map.constructMap().empty())
{
auto& cMap = const_cast<labelListList&>(map.constructMap());
cMap.resize_nocopy(UPstream::nProcs(map.comm()));
}
if (map.subMap().empty())
{
auto& cMap = const_cast<labelListList&>(map.subMap());
cMap.resize_nocopy(UPstream::nProcs(map.comm()));
}
// Get all restriction addressing.
labelList allRestrict(targetRestrictAddressing);
map.distribute(allRestrict);
@ -621,7 +709,8 @@ Foam::AMIInterpolation::AMIInterpolation
reverseTarget_(fineAMI.reverseTarget_),
lowWeightCorrection_(-1.0),
singlePatchProc_(fineAMI.singlePatchProc_),
comm_(fineAMI.comm_),
comm_(fineAMI.comm()), // use fineAMI geomComm if present, comm otherwise
geomComm_(),
srcMagSf_(),
srcAddress_(),
srcWeights_(),
@ -681,6 +770,26 @@ Foam::AMIInterpolation::AMIInterpolation
// Agglomerate addresses and weights
if (comm() != -1)
{
//Pout<< "** agglomerating srcAddress, tgtMap" << endl;
//if (fineAMI.tgtMapPtr_.valid())
//{
// const auto& fineTgtMap = fineAMI.tgtMapPtr_();
// Pout<< " fineAMI.tgtMapPtr_ comm:" << fineTgtMap.comm()
// << " procs:"
// << (
// fineTgtMap.comm() != -1
// ? UPstream::procID(fineTgtMap.comm())
// : labelList::null()
// )
// << endl;
//}
//else
//{
// Pout<< " NO fineAMI.tgtMapPtr_" << endl;
//}
//
agglomerate
(
fineAMI.tgtMapPtr_,
@ -696,9 +805,26 @@ Foam::AMIInterpolation::AMIInterpolation
srcWeights_,
srcWeightsSum_,
tgtMapPtr_,
comm_
comm()
);
//Pout<< "** agglomerating tgtAddress, srcMap" << endl;
//if (fineAMI.srcMapPtr_.valid())
//{
// const auto& fineSrcMap = fineAMI.srcMapPtr_();
// Pout<< " fineAMI.srcMapPtr_ comm:" << fineSrcMap.comm()
// << " procs:"
// << (
// fineSrcMap.comm() != -1
// ? UPstream::procID(fineSrcMap.comm())
// : labelList::null()
// )
// << endl;
//}
//else
//{
// Pout<< " NO fineAMI.srcMapPtr_" << endl;
//}
agglomerate
(
fineAMI.srcMapPtr_,
@ -714,9 +840,10 @@ Foam::AMIInterpolation::AMIInterpolation
tgtWeights_,
tgtWeightsSum_,
srcMapPtr_,
comm_
comm()
);
}
}
Foam::AMIInterpolation::AMIInterpolation(const AMIInterpolation& ami)
@ -726,6 +853,7 @@ Foam::AMIInterpolation::AMIInterpolation(const AMIInterpolation& ami)
lowWeightCorrection_(ami.lowWeightCorrection_),
singlePatchProc_(ami.singlePatchProc_),
comm_(ami.comm_),
geomComm_(ami.geomComm_), // ? steals communicator
srcMagSf_(ami.srcMagSf_),
srcAddress_(ami.srcAddress_),
srcWeights_(ami.srcWeights_),
@ -748,7 +876,7 @@ Foam::AMIInterpolation::AMIInterpolation(Istream& is)
reverseTarget_(readBool(is)),
lowWeightCorrection_(readScalar(is)),
singlePatchProc_(readLabel(is)),
comm_(readLabel(is)),
comm_(readLabel(is)), // either geomComm_ or comm_ from sending side
srcMagSf_(is),
srcAddress_(is),
@ -768,7 +896,10 @@ Foam::AMIInterpolation::AMIInterpolation(Istream& is)
upToDate_(readBool(is))
{
if (singlePatchProc_ == -1)
// Hopefully no need to stream geomComm_ since only used in processor
// agglomeration?
if (singlePatchProc_ == -1 && comm_ != -1)
{
srcMapPtr_.reset(new mapDistribute(is));
tgtMapPtr_.reset(new mapDistribute(is));
@ -816,6 +947,7 @@ bool Foam::AMIInterpolation::calculate
ttgtPatch0_.cref(tgtPatch);
}
// Note: use original communicator for statistics
label srcTotalSize = returnReduce
(
srcPatch.size(),
@ -842,7 +974,11 @@ bool Foam::AMIInterpolation::calculate
return false;
}
singlePatchProc_ = calcDistribution(srcPatch, tgtPatch);
// Calculate:
// - which processors have faces
// - allocates a communicator (geomComm_) for those
// - if it is only one processor that holds all faces
singlePatchProc_ = calcDistribution(srcPatch, tgtPatch, comm_, geomComm_);
Info<< indent << "AMI: Patch source faces: " << srcTotalSize << nl
<< indent << "AMI: Patch target faces: " << tgtTotalSize << nl;
@ -913,7 +1049,7 @@ void Foam::AMIInterpolation::append
newPtr->calculate(srcPatch, tgtPatch);
// If parallel then combine the mapDistribution and re-index
if (distributed())
if (distributed() && comm() != -1)
{
labelListList& srcSubMap = srcMapPtr_->subMap();
labelListList& srcConstructMap = srcMapPtr_->constructMap();
@ -1107,7 +1243,7 @@ void Foam::AMIInterpolation::normaliseWeights
conformal,
output,
lowWeightCorrection_,
comm_
comm()
);
normaliseWeights
@ -1120,7 +1256,7 @@ void Foam::AMIInterpolation::normaliseWeights
conformal,
output,
lowWeightCorrection_,
comm_
comm()
);
}
@ -1345,7 +1481,7 @@ bool Foam::AMIInterpolation::writeData(Ostream& os) const
<< token::SPACE<< reverseTarget()
<< token::SPACE<< lowWeightCorrection()
<< token::SPACE<< singlePatchProc()
<< token::SPACE<< comm()
<< token::SPACE<< comm() // either geomComm_ or comm_
<< token::SPACE<< srcMagSf()
<< token::SPACE<< srcAddress()
@ -1361,7 +1497,7 @@ bool Foam::AMIInterpolation::writeData(Ostream& os) const
<< token::SPACE<< upToDate();
if (distributed())
if (distributed() && comm() != -1)
{
os << token::SPACE<< srcMap()
<< token::SPACE<< tgtMap();

19
src/meshTools/AMIInterpolation/AMIInterpolation/AMIInterpolation.H
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2016-2023 OpenCFD Ltd.
Copyright (C) 2016-2024 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -80,6 +80,11 @@ public:
static bool cacheIntersections_;
//- localComm : 0 : all processors (backwards compatible)
//- localComm : 1 : only processors with patch faces
//- localComm : 2 : like 1 but add master processor always
static int localComm_;
protected:
@ -106,6 +111,10 @@ protected:
//- Communicator to use for parallel operations.
label comm_;
//- Communicator to use for geometry calculations. Not valid (-1) on
//- processors that do not have faces
autoPtr<UPstream::communicator> geomComm_;
// Source patch
@ -181,11 +190,15 @@ protected:
const primitivePatch& patch
) const;
//- Calculate if patches are on multiple processors
//- Calculate if patches are on multiple processors. Allocates
//- local communicator and returns -1 or processor containing all
//- faces
label calcDistribution
(
const primitivePatch& srcPatch,
const primitivePatch& tgtPatch
const primitivePatch& tgtPatch,
const label comm,
autoPtr<UPstream::communicator>& geomComm
) const;
//- Project points to surface

7
src/meshTools/AMIInterpolation/AMIInterpolation/AMIInterpolationI.H
View File

@ -103,9 +103,16 @@ inline Foam::label Foam::AMIInterpolation::singlePatchProc() const noexcept
inline Foam::label Foam::AMIInterpolation::comm() const
{
if (geomComm_.valid())
{
return geomComm_();
}
else
{
return comm_;
}
}
inline Foam::label Foam::AMIInterpolation::comm(const label newComm)

16
src/meshTools/AMIInterpolation/AMIInterpolation/AMIInterpolationTemplates.C
View File

@ -142,9 +142,15 @@ void Foam::AMIInterpolation::interpolateToTarget
result.setSize(tgtAddress_.size());
List<Type> work;
if (distributed())
if (distributed() && srcMapPtr_)
{
const mapDistribute& map = srcMapPtr_();
if (map.comm() == -1)
{
return;
}
work.resize_nocopy(map.constructSize());
SubList<Type>(work, fld.size()) = fld; // deep copy
map.distribute(work);
@ -203,9 +209,15 @@ void Foam::AMIInterpolation::interpolateToSource
result.setSize(srcAddress_.size());
List<Type> work;
if (distributed())
if (distributed() && tgtMapPtr_)
{
const mapDistribute& map = tgtMapPtr_();
if (map.comm() == -1)
{
return;
}
work.resize_nocopy(map.constructSize());
SubList<Type>(work, fld.size()) = fld; // deep copy
map.distribute(work);

16
src/meshTools/AMIInterpolation/AMIInterpolation/advancingFrontAMI/advancingFrontAMI.C
View File

@ -63,7 +63,7 @@ void Foam::advancingFrontAMI::checkPatches() const
}
if (requireMatch_)
if (requireMatch_ && comm() != -1)
{
const scalar maxBoundsError = 0.05;
@ -74,14 +74,14 @@ void Foam::advancingFrontAMI::checkPatches() const
bbSrc.min(),
minOp<point>(),
UPstream::msgType(),
comm_
comm()
);
Foam::reduce
(
bbSrc.max(),
maxOp<point>(),
UPstream::msgType(),
comm_
comm()
);
boundBox bbTgt(tgt.points(), tgt.meshPoints(), false);
Foam::reduce
@ -89,14 +89,14 @@ void Foam::advancingFrontAMI::checkPatches() const
bbTgt.min(),
minOp<point>(),
UPstream::msgType(),
comm_
comm()
);
Foam::reduce
(
bbTgt.max(),
maxOp<point>(),
UPstream::msgType(),
comm_
comm()
);
boundBox bbTgtInf(bbTgt);
@ -178,7 +178,7 @@ void Foam::advancingFrontAMI::createExtendedTgtPatch()
// Original faces from tgtPatch
// Note: in globalIndexing since might be remote
globalIndex globalTgtFaces(tgtPatch0().size(), comm_);
globalIndex globalTgtFaces(tgtPatch0().size(), comm());
distributeAndMergePatches
(
map,
@ -465,7 +465,7 @@ void Foam::advancingFrontAMI::triangulatePatch
void Foam::advancingFrontAMI::nonConformalCorrection()
{
if (!requireMatch_ && distributed())
if (!requireMatch_ && distributed() && comm() != -1)
{
scalarList newTgtMagSf(std::move(tgtMagSf_));
@ -587,7 +587,7 @@ bool Foam::advancingFrontAMI::calculate
{
// Create a representation of the target patch that covers the source
// patch
if (distributed())
if (distributed() && comm() != -1)
{
createExtendedTgtPatch();
}

53
src/meshTools/AMIInterpolation/AMIInterpolation/advancingFrontAMI/advancingFrontAMIParallelOps.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2018-2022 OpenCFD Ltd.
Copyright (C) 2018-2024 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -74,13 +74,13 @@ void Foam::advancingFrontAMI::distributePatches
List<labelList>& faceIDs
) const
{
faces.setSize(Pstream::nProcs(comm_));
points.setSize(Pstream::nProcs(comm_));
faceIDs.setSize(Pstream::nProcs(comm_));
faces.setSize(Pstream::nProcs(comm()));
points.setSize(Pstream::nProcs(comm()));
faceIDs.setSize(Pstream::nProcs(comm()));
PstreamBuffers pBufs(comm_);
PstreamBuffers pBufs(comm());
for (const int domain : Pstream::allProcs(comm_))
for (const int domain : Pstream::allProcs(comm()))
{
const labelList& sendElems = map.subMap()[domain];
@ -103,13 +103,13 @@ void Foam::advancingFrontAMI::distributePatches
}
if (domain == Pstream::myProcNo(comm_))
if (domain == Pstream::myProcNo(comm()))
{
// Do send/receive for myself
faces[domain] = subPatch.localFaces();
points[domain] = subPatch.localPoints();
faceIDs[domain] =
gi.toGlobal(Pstream::myProcNo(comm_), sendElems);
gi.toGlobal(Pstream::myProcNo(comm()), sendElems);
}
else
{
@ -118,7 +118,7 @@ void Foam::advancingFrontAMI::distributePatches
str
<< subPatch.localFaces()
<< subPatch.localPoints()
<< gi.toGlobal(Pstream::myProcNo(comm_), sendElems);
<< gi.toGlobal(Pstream::myProcNo(comm()), sendElems);
}
}
}
@ -127,11 +127,11 @@ void Foam::advancingFrontAMI::distributePatches
// Consume
for (const int domain : Pstream::allProcs(comm_))
for (const int domain : Pstream::allProcs(comm()))
{
const labelList& recvElems = map.constructMap()[domain];
if (domain != Pstream::myProcNo(comm_) && recvElems.size())
if (domain != Pstream::myProcNo(comm()) && recvElems.size())
{
UIPstream is(domain, pBufs);
@ -177,10 +177,10 @@ void Foam::advancingFrontAMI::distributeAndMergePatches
// My own data first
{
const labelList& faceIDs = allTgtFaceIDs[Pstream::myProcNo(comm_)];
const labelList& faceIDs = allTgtFaceIDs[Pstream::myProcNo(comm())];
SubList<label>(tgtFaceIDs, faceIDs.size()) = faceIDs;
const faceList& fcs = allFaces[Pstream::myProcNo(comm_)];
const faceList& fcs = allFaces[Pstream::myProcNo(comm())];
for (const face& f : fcs)
{
face& newF = tgtFaces[nFaces++];
@ -191,7 +191,7 @@ void Foam::advancingFrontAMI::distributeAndMergePatches
}
}
const pointField& pts = allPoints[Pstream::myProcNo(comm_)];
const pointField& pts = allPoints[Pstream::myProcNo(comm())];
for (const point& pt: pts)
{
tgtPoints[nPoints++] = pt;
@ -202,7 +202,7 @@ void Foam::advancingFrontAMI::distributeAndMergePatches
// Other proc data follows
forAll(allFaces, proci)
{
if (proci != Pstream::myProcNo(comm_))
if (proci != Pstream::myProcNo(comm()))
{
const labelList& faceIDs = allTgtFaceIDs[proci];
SubList<label>(tgtFaceIDs, faceIDs.size(), nFaces) = faceIDs;
@ -258,12 +258,19 @@ Foam::autoPtr<Foam::mapDistribute> Foam::advancingFrontAMI::calcProcMap
const primitivePatch& tgtPatch
) const
{
if (comm() == -1)
{
FatalErrorInFunction
<< "Processor " << UPstream::myProcNo(UPstream::worldComm)
<< " not in communicator " << comm() << exit(FatalError);
}
// Get decomposition of patch
List<treeBoundBoxList> procBb(Pstream::nProcs(comm_));
List<treeBoundBoxList> procBb(Pstream::nProcs(comm()));
if (srcPatch.size())
{
procBb[Pstream::myProcNo(comm_)] =
procBb[Pstream::myProcNo(comm())] =
AABBTree<face>
(
srcPatch.localFaces(),
@ -273,10 +280,10 @@ Foam::autoPtr<Foam::mapDistribute> Foam::advancingFrontAMI::calcProcMap
}
else
{
procBb[Pstream::myProcNo(comm_)] = treeBoundBoxList();
procBb[Pstream::myProcNo(comm())] = treeBoundBoxList();
}
Pstream::allGatherList(procBb, UPstream::msgType(), comm_);
Pstream::allGatherList(procBb, UPstream::msgType(), comm());
if (debug)
{
@ -296,10 +303,10 @@ Foam::autoPtr<Foam::mapDistribute> Foam::advancingFrontAMI::calcProcMap
{
// Per processor indices into all segments to send
List<DynamicList<label>> dynSendMap(Pstream::nProcs(comm_));
List<DynamicList<label>> dynSendMap(Pstream::nProcs(comm()));
// Work array - whether processor bb overlaps the face bounds
boolList procBbOverlaps(Pstream::nProcs(comm_));
boolList procBbOverlaps(Pstream::nProcs(comm()));
forAll(faces, facei)
{
@ -321,7 +328,7 @@ Foam::autoPtr<Foam::mapDistribute> Foam::advancingFrontAMI::calcProcMap
}
// Convert dynamicList to labelList
sendMap.setSize(Pstream::nProcs(comm_));
sendMap.setSize(Pstream::nProcs(comm()));
forAll(sendMap, proci)
{
sendMap[proci].transfer(dynSendMap[proci]);
@ -344,7 +351,7 @@ Foam::autoPtr<Foam::mapDistribute> Foam::advancingFrontAMI::calcProcMap
std::move(sendMap),
false, //subHasFlip
false, //constructHasFlip
comm_
comm()
);
}

16
src/meshTools/AMIInterpolation/AMIInterpolation/faceAreaWeightAMI/faceAreaWeightAMI.C
View File

@ -711,9 +711,9 @@ bool Foam::faceAreaWeightAMI::calculate
tgtWeights_[i].transfer(tgtWght[i]);
}
if (distributed())
if (distributed() && comm() != -1)
{
const label myRank = UPstream::myProcNo(comm_);
const label myRank = UPstream::myProcNo(comm());
// Allocate unique tag for all comms
const int oldTag = UPstream::incrMsgType();
@ -721,8 +721,8 @@ bool Foam::faceAreaWeightAMI::calculate
const primitivePatch& tgtPatch0 = this->tgtPatch0();
// Create global indexing for each original patch
globalIndex globalSrcFaces(srcPatch0.size(), comm_);
globalIndex globalTgtFaces(tgtPatch0.size(), comm_);
globalIndex globalSrcFaces(srcPatch0.size(), comm());
globalIndex globalTgtFaces(tgtPatch0.size(), comm());
for (labelList& addressing : srcAddress_)
{
@ -754,7 +754,7 @@ bool Foam::faceAreaWeightAMI::calculate
ListOps::appendEqOp<label>(),
flipOp(), // flip operation
UPstream::msgType()+77431,
comm_
comm()
);
mapDistributeBase::distribute
@ -771,7 +771,7 @@ bool Foam::faceAreaWeightAMI::calculate
ListOps::appendEqOp<scalar>(),
flipOp(),
UPstream::msgType()+77432,
comm_
comm()
);
// Note: using patch face areas calculated by the AMI method
@ -787,7 +787,7 @@ bool Foam::faceAreaWeightAMI::calculate
tgtAddress_,
cMapSrc,
UPstream::msgType()+77433,
comm_
comm()
)
);
@ -800,7 +800,7 @@ bool Foam::faceAreaWeightAMI::calculate
srcAddress_,
cMapTgt,
UPstream::msgType()+77434,
comm_
comm()
)
);

16
src/meshTools/AMIInterpolation/AMIInterpolation/faceAreaWeightAMI2D/faceAreaWeightAMI2D.C
View File

@ -425,9 +425,9 @@ bool Foam::faceAreaWeightAMI2D::calculate
tgtWeights_[i].transfer(tgtWght[i]);
}
if (distributed())
if (distributed() && comm() != -1)
{
const label myRank = UPstream::myProcNo(comm_);
const label myRank = UPstream::myProcNo(comm());
// Allocate unique tag for all comms
const int oldTag = UPstream::incrMsgType();
@ -435,8 +435,8 @@ bool Foam::faceAreaWeightAMI2D::calculate
const primitivePatch& tgtPatch0 = this->tgtPatch0();
// Create global indexing for each original patch
const globalIndex globalSrcFaces(srcPatch0.size(), comm_);
const globalIndex globalTgtFaces(tgtPatch0.size(), comm_);
const globalIndex globalSrcFaces(srcPatch0.size(), comm());
const globalIndex globalTgtFaces(tgtPatch0.size(), comm());
for (labelList& addressing : srcAddress_)
{
@ -468,7 +468,7 @@ bool Foam::faceAreaWeightAMI2D::calculate
ListOps::appendEqOp<label>(),
flipOp(), // flip operation
UPstream::msgType()+77431,
comm_
comm()
);
mapDistributeBase::distribute
@ -485,7 +485,7 @@ bool Foam::faceAreaWeightAMI2D::calculate
ListOps::appendEqOp<scalar>(),
flipOp(), // flip operation
UPstream::msgType()+77432,
comm_
comm()
);
// Note: using patch face areas calculated by the AMI method
@ -501,7 +501,7 @@ bool Foam::faceAreaWeightAMI2D::calculate
tgtAddress_,
cMapSrc,
UPstream::msgType()+77433,
comm_
comm()
)
);
@ -514,7 +514,7 @@ bool Foam::faceAreaWeightAMI2D::calculate
srcAddress_,
cMapTgt,
UPstream::msgType()+77434,
comm_
comm()
)
);

34
src/meshTools/AMIInterpolation/AMIInterpolation/nearestFaceAMI/nearestFaceAMI.C
View File

@ -5,7 +5,7 @@
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2020-2022 OpenCFD Ltd.
Copyright (C) 2020-2024 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -47,15 +47,15 @@ Foam::autoPtr<Foam::mapDistribute> Foam::nearestFaceAMI::calcFaceMap
) const
{
// Generate the list of processor bounding boxes for tgtPatch
List<boundBox> procBbs(Pstream::nProcs());
procBbs[Pstream::myProcNo()] =
boundBox(tgtPatch.points(), tgtPatch.meshPoints(), true);
Pstream::allGatherList(procBbs);
List<boundBox> procBbs(Pstream::nProcs(comm()));
procBbs[Pstream::myProcNo(comm())] =
boundBox(tgtPatch.points(), tgtPatch.meshPoints(), false);
Pstream::allGatherList(procBbs, UPstream::msgType(), comm());
// Identify which of my local src faces intersect with each processor
// tgtPatch bb within the current match's search distance
const pointField& srcCcs = srcPatch.faceCentres();
List<DynamicList<label>> dynSendMap(Pstream::nProcs());
List<DynamicList<label>> dynSendMap(Pstream::nProcs(comm()));
forAll(localInfo, srcFacei)
{
@ -66,7 +66,7 @@ Foam::autoPtr<Foam::mapDistribute> Foam::nearestFaceAMI::calcFaceMap
forAll(procBbs, proci)
{
if (proci != Pstream::myProcNo())
if (proci != Pstream::myProcNo(comm()))
{
if (procBbs[proci].overlaps(srcCcs[srcFacei], r2))
{
@ -77,7 +77,7 @@ Foam::autoPtr<Foam::mapDistribute> Foam::nearestFaceAMI::calcFaceMap
}
// Convert dynamicList to labelList
labelListList sendMap(Pstream::nProcs());
labelListList sendMap(Pstream::nProcs(comm()));
forAll(sendMap, proci)
{
sendMap[proci].transfer(dynSendMap[proci]);
@ -89,7 +89,13 @@ Foam::autoPtr<Foam::mapDistribute> Foam::nearestFaceAMI::calcFaceMap
}
}
return autoPtr<mapDistribute>::New(std::move(sendMap));
return autoPtr<mapDistribute>::New
(
std::move(sendMap),
false,
false,
comm()
);
}
@ -108,10 +114,10 @@ Foam::autoPtr<Foam::mapDistribute> Foam::nearestFaceAMI::calcDistributed
}
// Create global indexing for tgtPatch
globalIndex globalTgtCells(tgt.size());
globalIndex globalTgtCells(tgt.size(), comm());
const label myRank = UPstream::myProcNo(comm_);
const label myRank = UPstream::myProcNo(comm());
// First pass
@ -205,7 +211,7 @@ Foam::autoPtr<Foam::mapDistribute> Foam::nearestFaceAMI::calcDistributed
nearestEqOp(),
identityOp(), // No flipping
UPstream::msgType(),
comm_
comm()
);
@ -234,7 +240,7 @@ Foam::autoPtr<Foam::mapDistribute> Foam::nearestFaceAMI::calcDistributed
srcToTgtAddr,
cMap,
UPstream::msgType(),
comm_
comm()
);
}
@ -297,7 +303,7 @@ bool Foam::nearestFaceAMI::calculate
// TODO: implement symmetric calculation controls; assume yes for now
bool symmetric_ = true;
if (this->distributed())
if (this->distributed() && comm() != -1)
{
tgtMapPtr_ =
calcDistributed

4
src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicACMIGAMGInterfaceField/cyclicACMIGAMGInterfaceField.C
View File

@ -184,7 +184,7 @@ void Foam::cyclicACMIGAMGInterfaceField::initInterfaceMatrixUpdate
: cyclicACMIInterface_.neighbPatch().AMI()
);
if (AMI.distributed())
if (AMI.distributed() && AMI.comm() != -1)
{
DebugPout<< "cyclicACMIFvPatchField::initInterfaceMatrixUpdate() :"
<< " interface:" << cyclicACMIInterface_.index()
@ -277,7 +277,7 @@ void Foam::cyclicACMIGAMGInterfaceField::updateInterfaceMatrix
<< endl;
if (AMI.distributed())
if (AMI.distributed() && AMI.comm() != -1)
{
const auto& map =
(

4
src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicAMIGAMGInterfaceField/cyclicAMIGAMGInterfaceField.C
View File

@ -184,7 +184,7 @@ void Foam::cyclicAMIGAMGInterfaceField::initInterfaceMatrixUpdate
: cyclicAMIInterface_.neighbPatch().AMI()
);
if (AMI.distributed())
if (AMI.distributed() && AMI.comm() != -1)
{
//DebugPout<< "cyclicAMIFvPatchField::initInterfaceMatrixUpdate() :"
// << " interface:" << cyclicAMIInterface_.index()
@ -284,7 +284,7 @@ void Foam::cyclicAMIGAMGInterfaceField::updateInterfaceMatrix
// << " AMI low-weight:" << AMI.applyLowWeightCorrection()
// << endl;
if (AMI.distributed())
if (AMI.distributed() && AMI.comm() != -1)
{
if (commsType != UPstream::commsTypes::nonBlocking)
{

315
src/meshTools/AMIInterpolation/GAMG/interfaces/cyclicACMIGAMGInterface/cyclicACMIGAMGInterface.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2013-2016 OpenFOAM Foundation
Copyright (C) 2019,2023 OpenCFD Ltd.
Copyright (C) 2019-2025 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -84,7 +84,8 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
reverseT_
(
refCast<const cyclicACMILduInterface>(fineInterface).reverseT()
)
),
myProcNo_(-1)
{
const auto& fineCyclicACMIInterface =
refCast<const cyclicACMILduInterface>(fineInterface);
@ -190,13 +191,24 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
neighbPatchID_(readLabel(is)),
owner_(readBool(is)),
forwardT_(is),
reverseT_(is)
reverseT_(is),
myProcNo_(-1)
{
const bool hasAMI(readBool(is));
if (hasAMI)
{
amiPtr_.reset(new AMIPatchToPatchInterpolation(is));
// Store originating ranks locally - used when processor agglomerating
// onto a processor that wasn't in the communicator originally (since
// it had no faces)
const label comm = AMI().comm();
if (comm != -1)
{
is >> myProcNo_;
}
}
}
@ -244,27 +256,64 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
reverseT_
(
refCast<const cyclicACMILduInterface>(fineInterface).reverseT()
)
),
myProcNo_(-1)
{
const auto& fineCyclicACMIInterface =
refCast<const cyclicACMIGAMGInterface>(fineInterface);
if (fineCyclicACMIInterface.amiPtr_)
if (!owner_)
{
const auto& AMI = const_cast<AMIPatchToPatchInterpolation&>
(
fineCyclicACMIInterface.AMI()
);
label singlePatchProc = AMI.singlePatchProc();
return;
}
// Get some sizes
label nSrc = 0;
label nTgt = 0;
// Get stats, sizes from the input interfaces. For the global settings
// the problem is that the
// local processor might not have any valid interfaces so here just
// collect and do a global reduction afterwards.
// Structure to pack all. First element is used to decide who has the
// valid AMI.
typedef
Tuple2
<
label,
Tuple2
<
Tuple2
<
FixedList<bool, 4>,
scalar
>,
label
>
> AMIType;
AMIType globalInfo;
FixedList<bool, 4>& bools = globalInfo.second().first().first();
// Define aliases to make our life easier
label& firstValidAMI = globalInfo.first();
bool& requireMatch = bools[0];
bool& reverseTarget = bools[1];
bool& srcHasFlip = bools[2];
bool& tgtHasFlip = bools[3];
scalar& lowWeightCorrection = globalInfo.second().first().second();
label& singlePatchProc = globalInfo.second().second();
// Initialise all global variables
firstValidAMI = labelMax;
requireMatch = false;
reverseTarget = false;
srcHasFlip = false;
tgtHasFlip = false;
lowWeightCorrection = -1;
singlePatchProc = -1;
// Initialise all local variables
bool hasSrcMagSf = false;
bool hasSrcCentroids = false;
bool hasTgtMagSf = false;
label nSrc = 0;
label nTgt = 0;
forAll(allInterfaces, inti)
{
@ -272,7 +321,31 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
{
const auto& intf =
refCast<const cyclicACMIGAMGInterface>(allInterfaces[inti]);
if (!intf.amiPtr_)
{
continue;
}
if (firstValidAMI == labelMax)
{
firstValidAMI = inti;
}
const auto& AMI = intf.AMI();
if (AMI.distributed() && AMI.comm() != -1)
{
singlePatchProc = -1;
srcHasFlip =
srcHasFlip || AMI.srcMap().constructHasFlip();
tgtHasFlip =
tgtHasFlip || AMI.tgtMap().constructHasFlip();
}
requireMatch = AMI.requireMatch();
reverseTarget = AMI.reverseTarget();
lowWeightCorrection = AMI.lowWeightCorrection();
nSrc += AMI.srcAddress().size();
nTgt += AMI.tgtAddress().size();
@ -313,6 +386,39 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
}
// Reduce global information in case one of the coarse ranks does not
// have an input AMI to get data from. Could use minFirstEqOp from Tuple2
// instead ...
Pstream::combineReduce
(
globalInfo,
[](AMIType& x, const AMIType& y)
{
if (y.first() < x.first())
{
x = y;
}
},
Pstream::msgType(),
coarseComm
);
DebugPout
<< "Input amis :"
<< " singlePatchProc:" << singlePatchProc
<< " srcHasFlip:" << srcHasFlip
<< " tgtHasFlip:" << tgtHasFlip
<< " requireMatch:" << requireMatch
<< " reverseTarget:" << reverseTarget
<< " lowWeightCorrection:" << lowWeightCorrection
<< " hasSrcMagSf:" << hasSrcMagSf
<< " hasSrcCentroids:" << hasSrcCentroids
<< " hasTgtMagSf:" << hasTgtMagSf
<< " nSrc:" << nSrc
<< " nTgt:" << nTgt
<< endl;
labelListList srcAddress;
scalarListList srcWeights;
scalarList srcMagSf;
@ -330,8 +436,81 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
// Map to send tgt side data to src side
autoPtr<mapDistribute> tgtToSrcMap;
if (AMI.distributed())
if (singlePatchProc == -1)
{
// Find ranks that agglomerate together
const label myAgglom = UPstream::myProcNo(coarseComm);
// Per input map either -1 or the index in the maps that is local
// data.
labelList localRanks(allInterfaces.size(), -1);
// From rank in coarse communicator back to rank in original (fine)
// communicator.
labelListList newToOldRanks;
{
// Pass 1: count number of valid maps
label nOldRanks = 0;
forAll(allInterfaces, inti)
{
if (allInterfaces.set(inti))
{
const auto& intf = refCast<const cyclicACMIGAMGInterface>
(
allInterfaces[inti]
);
if (!intf.amiPtr_ || intf.AMI().comm() == -1)
{
continue;
}
nOldRanks++;
}
}
// Pass 2: collect
DynamicList<label> oldRanks(nOldRanks);
forAll(allInterfaces, inti)
{
if (allInterfaces.set(inti))
{
const auto& intf = refCast<const cyclicACMIGAMGInterface>
(
allInterfaces[inti]
);
if (!intf.amiPtr_ || intf.AMI().comm() == -1)
{
continue;
}
label fineRank = -1;
if (intf.myProcNo() == -1)
{
// The interface was already local so got never
// sent across so myProcNo_ is never set ...
fineRank = UPstream::myProcNo(intf.AMI().comm());
}
else
{
fineRank = intf.myProcNo();
}
oldRanks.append(fineRank);
localRanks[inti] = fineRank;
}
}
// Pull individual parts together - this is the only communication
// needed.
newToOldRanks = Pstream::listGatherValues
(
labelList(std::move(oldRanks)),
coarseComm
);
Pstream::broadcast(newToOldRanks, coarseComm);
}
// Create combined maps
UPtrList<const mapDistribute> srcMaps(allInterfaces.size());
UPtrList<const mapDistribute> tgtMaps(allInterfaces.size());
@ -343,27 +522,22 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
(
allInterfaces[inti]
);
if (!intf.amiPtr_)
{
// Should not be in allInterfaces?
continue;
}
const auto& AMI = intf.AMI();
if (AMI.comm() != -1)
{
srcMaps.set(inti, &AMI.srcMap());
tgtMaps.set(inti, &AMI.tgtMap());
}
}
// Find ranks that agglomerate together
const label myAgglom =
procAgglomMap[UPstream::myProcNo(AMI.comm())];
// Invert procAgglomMap
const labelListList newToOldRanks
(
invertOneToMany
(
UPstream::nProcs(coarseComm),
procAgglomMap
)
);
const labelList& localRanks = newToOldRanks[myAgglom];
}
// Offsets for slots into results of srcToTgtMap
@ -375,7 +549,7 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
new mapDistribute
(
srcMaps,
localRanks, // per src map which rank it is from
localRanks, // per src map which rank represents local data
coarseComm,
newToOldRanks, // destination rank to source ranks
srcStartOfLocal,
@ -383,6 +557,7 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
)
);
// Assemble tgtAddress
tgtAddress.setSize(nTgt);
if (tgtAddress.size())
@ -392,11 +567,16 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
{
if (allInterfaces.set(inti))
{
const auto& intf =
refCast<const cyclicACMIGAMGInterface>
const auto& intf = refCast<const cyclicACMIGAMGInterface>
(
allInterfaces[inti]
);
if (!intf.amiPtr_)
{
continue;
}
const auto& AMI = intf.AMI();
const auto& tgtSlots = AMI.tgtAddress();
const label localSize =
@ -416,7 +596,7 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
localSize,
srcStartOfLocal[inti],
srcCompactMaps[inti],
AMI.srcMap().constructHasFlip() //hasFlip
srcHasFlip //hasFlip
);
for (const label slot : newSlots)
@ -459,6 +639,7 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
)
);
// Assemble srcAddress
srcAddress.setSize(nSrc);
if (srcAddress.size())
@ -468,11 +649,16 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
{
if (allInterfaces.set(inti))
{
const auto& intf =
refCast<const cyclicACMIGAMGInterface>
const auto& intf = refCast<const cyclicACMIGAMGInterface>
(
allInterfaces[inti]
);
if (!intf.amiPtr_)
{
continue;
}
const auto& AMI = intf.AMI();
const auto& srcSlots = AMI.srcAddress();
const label localSize =
@ -491,7 +677,7 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
localSize,
tgtStartOfLocal[inti],
tgtCompactMaps[inti],
AMI.tgtMap().constructHasFlip() //hasFlip
tgtHasFlip
);
for (const label slot : newSlots)
@ -543,10 +729,14 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
}
}
if (!usesRemote)
// We can't have a single rank become fully-local since we
// expect singlePatchProc to be synchronised. So make sure all
// have become local
if (!returnReduceOr(usesRemote, coarseComm))
{
//Pout<< "** making fully local on new rank "
// << myAgglom << " in comm:" << coarseComm << endl;
DebugPout<< "** making fully local on new rank "
<< myAgglom << " in comm:" << coarseComm << endl;
singlePatchProc = myAgglom;
srcToTgtMap.clear();
tgtToSrcMap.clear();
@ -570,6 +760,12 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
(
allInterfaces[inti]
);
if (!intf.amiPtr_)
{
continue;
}
const auto& AMI = intf.AMI();
const auto& srcA = AMI.srcAddress();
@ -625,6 +821,12 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
(
allInterfaces[inti]
);
if (!intf.amiPtr_)
{
continue;
}
const auto& AMI = intf.AMI();
const auto& srcA = AMI.srcAddress();
@ -672,9 +874,9 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
(
new AMIPatchToPatchInterpolation
(
AMI.requireMatch(),
AMI.reverseTarget(),
AMI.lowWeightCorrection()
requireMatch,
reverseTarget,
lowWeightCorrection
)
);
amiPtr_().comm(coarseComm),
@ -691,14 +893,18 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
amiPtr_().srcMagSf() = std::move(srcMagSf);
amiPtr_().srcCentroids() = std::move(srcCentroids);
amiPtr_().tgtMagSf() = std::move(tgtMagSf);
}
//Pout<< "** constructed new ami:"
// << " comm:" << amiPtr_().comm()
// << " srcMap.comm:" << amiPtr_().srcMap().comm()
// << " tgtMap.comm:" << amiPtr_().tgtMap().comm()
// << endl;
}
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
Foam::tmp<Foam::labelField>
Foam::cyclicACMIGAMGInterface::internalFieldTransfer
Foam::tmp<Foam::labelField> Foam::cyclicACMIGAMGInterface::internalFieldTransfer
(
const Pstream::commsTypes commsType,
const labelUList& iF
@ -736,6 +942,15 @@ void Foam::cyclicACMIGAMGInterface::write(Ostream& os) const
{
os << token::SPACE;
AMI().writeData(os);
// Write processors in communicator
const label comm = AMI().comm();
if (comm != -1)
{
os << token::SPACE
<< UPstream::myProcNo(comm);
}
}
}

15
src/meshTools/AMIInterpolation/GAMG/interfaces/cyclicACMIGAMGInterface/cyclicACMIGAMGInterface.H
View File

@ -35,8 +35,8 @@ SourceFiles
\*---------------------------------------------------------------------------*/
#ifndef cyclicACMIGAMGInterface_H
#define cyclicACMIGAMGInterface_H
#ifndef Foam_cyclicACMIGAMGInterface_H
#define Foam_cyclicACMIGAMGInterface_H
#include "GAMGInterface.H"
#include "cyclicACMILduInterface.H"
@ -68,6 +68,11 @@ class cyclicACMIGAMGInterface
//- AMI interface
autoPtr<AMIPatchToPatchInterpolation> amiPtr_;
//- rank in AMI with respect to parent communicator.
//- Used for combining contributions from different processors if the
//- destination processor was not in the original set.
label myProcNo_;
// Private Member Functions
@ -213,6 +218,12 @@ public:
return reverseT_;
}
//- -1 or old local rank
virtual label myProcNo() const
{
return myProcNo_;
}
// I/O

317
src/meshTools/AMIInterpolation/GAMG/interfaces/cyclicAMIGAMGInterface/cyclicAMIGAMGInterface.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2016 OpenFOAM Foundation
Copyright (C) 2019,2023 OpenCFD Ltd.
Copyright (C) 2019-2025 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -84,7 +84,8 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
reverseT_
(
refCast<const cyclicAMILduInterface>(fineInterface).reverseT()
)
),
myProcNo_(-1)
{
const auto& fineCyclicAMIInterface =
refCast<const cyclicAMILduInterface>(fineInterface);
@ -178,7 +179,8 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
const auto& AMI = amiPtr_();
if (debug & 2)
if (debug & 2 && AMI.comm() != -1)
{
const auto oldWarnComm = UPstream::commWarn(AMI.comm());
@ -204,7 +206,7 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
const auto& weights = AMI.srcWeights();
labelList globalIDs(identity(nbrSize));
if (AMI.distributed())
if (AMI.distributed() && AMI.comm() != -1)
{
const auto& map = AMI.tgtMap();
forAll(map.subMap(), proci)
@ -254,7 +256,7 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
const auto& weights = AMI.tgtWeights();
labelList globalIDs(identity(this->size()));
if (AMI.distributed())
if (AMI.distributed() && AMI.comm() != -1)
{
const auto& map = AMI.srcMap();
forAll(map.subMap(), proci)
@ -313,13 +315,24 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
neighbPatchID_(readLabel(is)),
owner_(readBool(is)),
forwardT_(is),
reverseT_(is)
reverseT_(is),
myProcNo_(-1)
{
const bool hasAMI(readBool(is));
if (hasAMI)
{
amiPtr_.reset(new AMIPatchToPatchInterpolation(is));
// Store originating ranks locally - used when processor agglomerating
// onto a processor that wasn't in the communicator originally (since
// it had no faces)
const label comm = AMI().comm();
if (comm != -1)
{
is >> myProcNo_;
}
}
}
@ -367,28 +380,64 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
reverseT_
(
refCast<const cyclicAMILduInterface>(fineInterface).reverseT()
)
),
myProcNo_(-1)
{
const auto& fineCyclicAMIInterface =
refCast<const cyclicAMIGAMGInterface>(fineInterface);
if (fineCyclicAMIInterface.amiPtr_)
if (!owner_)
{
const auto& fineAMI = const_cast<AMIPatchToPatchInterpolation&>
(
fineCyclicAMIInterface.AMI()
);
label singlePatchProc = fineAMI.singlePatchProc();
return;
}
// Get stats, sizes from the input interfaces. For the global settings
// the problem is that the
// local processor might not have any valid interfaces so here just
// collect and do a global reduction afterwards.
// Get some sizes
label nSrc = 0;
label nTgt = 0;
// Structure to pack all. First element is used to decide who has the
// valid AMI.
typedef
Tuple2
<
label,
Tuple2
<
Tuple2
<
FixedList<bool, 4>,
scalar
>,
label
>
> AMIType;
AMIType globalInfo;
FixedList<bool, 4>& bools = globalInfo.second().first().first();
// Define aliases to make our life easier
label& firstValidAMI = globalInfo.first();
bool& requireMatch = bools[0];
bool& reverseTarget = bools[1];
bool& srcHasFlip = bools[2];
bool& tgtHasFlip = bools[3];
scalar& lowWeightCorrection = globalInfo.second().first().second();
label& singlePatchProc = globalInfo.second().second();
// Initialise all global variables
firstValidAMI = labelMax;
requireMatch = false;
reverseTarget = false;
srcHasFlip = false;
tgtHasFlip = false;
lowWeightCorrection = -1;
singlePatchProc = -1;
// Initialise all local variables
bool hasSrcMagSf = false;
bool hasSrcCentroids = false;
bool hasTgtMagSf = false;
label nSrc = 0;
label nTgt = 0;
forAll(allInterfaces, inti)
{
@ -396,7 +445,31 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
{
const auto& intf =
refCast<const cyclicAMIGAMGInterface>(allInterfaces[inti]);
if (!intf.amiPtr_)
{
continue;
}
if (firstValidAMI == labelMax)
{
firstValidAMI = inti;
}
const auto& AMI = intf.AMI();
if (AMI.distributed() && AMI.comm() != -1)
{
singlePatchProc = -1;
srcHasFlip =
srcHasFlip || AMI.srcMap().constructHasFlip();
tgtHasFlip =
tgtHasFlip || AMI.tgtMap().constructHasFlip();
}
requireMatch = AMI.requireMatch();
reverseTarget = AMI.reverseTarget();
lowWeightCorrection = AMI.lowWeightCorrection();
nSrc += AMI.srcAddress().size();
nTgt += AMI.tgtAddress().size();
@ -437,6 +510,39 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
}
// Reduce global information in case one of the coarse ranks does not
// have an input AMI to get data from. Could use minFirstEqOp from Tuple2
// instead ...
Pstream::combineReduce
(
globalInfo,
[](AMIType& x, const AMIType& y)
{
if (y.first() < x.first())
{
x = y;
}
},
Pstream::msgType(),
coarseComm
);
DebugPout
<< "Input amis :"
<< " singlePatchProc:" << singlePatchProc
<< " srcHasFlip:" << srcHasFlip
<< " tgtHasFlip:" << tgtHasFlip
<< " requireMatch:" << requireMatch
<< " reverseTarget:" << reverseTarget
<< " lowWeightCorrection:" << lowWeightCorrection
<< " hasSrcMagSf:" << hasSrcMagSf
<< " hasSrcCentroids:" << hasSrcCentroids
<< " hasTgtMagSf:" << hasTgtMagSf
<< " nSrc:" << nSrc
<< " nTgt:" << nTgt
<< endl;
labelListList srcAddress;
scalarListList srcWeights;
scalarList srcMagSf;
@ -454,8 +560,81 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
// Map to send tgt side data to src side
autoPtr<mapDistribute> tgtToSrcMap;
if (fineAMI.distributed())
if (singlePatchProc == -1)
{
// Find ranks that agglomerate together
const label myAgglom = UPstream::myProcNo(coarseComm);
// Per input map either -1 or the index in the maps that is local
// data.
labelList localRanks(allInterfaces.size(), -1);
// From rank in coarse communicator back to rank in original (fine)
// communicator.
labelListList newToOldRanks;
{
// Pass 1: count number of valid maps
label nOldRanks = 0;
forAll(allInterfaces, inti)
{
if (allInterfaces.set(inti))
{
const auto& intf = refCast<const cyclicAMIGAMGInterface>
(
allInterfaces[inti]
);
if (!intf.amiPtr_ || intf.AMI().comm() == -1)
{
continue;
}
nOldRanks++;
}
}
// Pass 2: collect
DynamicList<label> oldRanks(nOldRanks);
forAll(allInterfaces, inti)
{
if (allInterfaces.set(inti))
{
const auto& intf = refCast<const cyclicAMIGAMGInterface>
(
allInterfaces[inti]
);
if (!intf.amiPtr_ || intf.AMI().comm() == -1)
{
continue;
}
label fineRank = -1;
if (intf.myProcNo() == -1)
{
// The interface was already local so got never
// sent across so myProcNo_ is never set ...
fineRank = UPstream::myProcNo(intf.AMI().comm());
}
else
{
fineRank = intf.myProcNo();
}
oldRanks.append(fineRank);
localRanks[inti] = fineRank;
}
}
// Pull individual parts together - this is the only communication
// needed.
newToOldRanks = Pstream::listGatherValues
(
labelList(std::move(oldRanks)),
coarseComm
);
Pstream::broadcast(newToOldRanks, coarseComm);
}
// Create combined maps
UPtrList<const mapDistribute> srcMaps(allInterfaces.size());
UPtrList<const mapDistribute> tgtMaps(allInterfaces.size());
@ -467,27 +646,22 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
(
allInterfaces[inti]
);
if (!intf.amiPtr_)
{
// Should not be in allInterfaces?
continue;
}
const auto& AMI = intf.AMI();
if (AMI.comm() != -1)
{
srcMaps.set(inti, &AMI.srcMap());
tgtMaps.set(inti, &AMI.tgtMap());
}
}
// Find ranks that agglomerate together
const label myAgglom =
procAgglomMap[UPstream::myProcNo(fineAMI.comm())];
// Invert procAgglomMap
const labelListList newToOldRanks
(
invertOneToMany
(
UPstream::nProcs(coarseComm),
procAgglomMap
)
);
const labelList& localRanks = newToOldRanks[myAgglom];
}
// Offsets for slots into results of srcToTgtMap
@ -499,7 +673,7 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
new mapDistribute
(
srcMaps,
localRanks, // per src map which rank it is from
localRanks, // per src map which rank represents local data
coarseComm,
newToOldRanks, // destination rank to source ranks
srcStartOfLocal,
@ -507,6 +681,7 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
)
);
// Assemble tgtAddress
tgtAddress.setSize(nTgt);
if (tgtAddress.size())
@ -520,6 +695,12 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
(
allInterfaces[inti]
);
if (!intf.amiPtr_)
{
continue;
}
const auto& AMI = intf.AMI();
const auto& tgtSlots = AMI.tgtAddress();
const label localSize =
@ -539,7 +720,7 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
localSize,
srcStartOfLocal[inti],
srcCompactMaps[inti],
AMI.srcMap().constructHasFlip() //hasFlip
srcHasFlip //hasFlip
);
for (const label slot : newSlots)
@ -582,6 +763,7 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
)
);
// Assemble srcAddress
srcAddress.setSize(nSrc);
if (srcAddress.size())
@ -595,6 +777,12 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
(
allInterfaces[inti]
);
if (!intf.amiPtr_)
{
continue;
}
const auto& AMI = intf.AMI();
const auto& srcSlots = AMI.srcAddress();
const label localSize =
@ -613,7 +801,7 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
localSize,
tgtStartOfLocal[inti],
tgtCompactMaps[inti],
AMI.tgtMap().constructHasFlip() //hasFlip
tgtHasFlip
);
for (const label slot : newSlots)
@ -665,10 +853,14 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
}
}
if (!usesRemote)
// We can't have a single rank become fully-local since we
// expect singlePatchProc to be synchronised. So make sure all
// have become local
if (!returnReduceOr(usesRemote, coarseComm))
{
//Pout<< "** making fully local on new rank "
// << myAgglom << " in comm:" << coarseComm << endl;
DebugPout<< "** making fully local on new rank "
<< myAgglom << " in comm:" << coarseComm << endl;
singlePatchProc = myAgglom;
srcToTgtMap.clear();
tgtToSrcMap.clear();
@ -692,6 +884,12 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
(
allInterfaces[inti]
);
if (!intf.amiPtr_)
{
continue;
}
const auto& AMI = intf.AMI();
const auto& srcA = AMI.srcAddress();
@ -747,6 +945,13 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
(
allInterfaces[inti]
);
if (!intf.amiPtr_)
{
continue;
}
const auto oldWarnComm = UPstream::commWarn(AMI.comm());
const auto& AMI = intf.AMI();
const auto& srcA = AMI.srcAddress();
@ -794,9 +999,9 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
(
new AMIPatchToPatchInterpolation
(
fineAMI.requireMatch(),
fineAMI.reverseTarget(),
fineAMI.lowWeightCorrection()
requireMatch,
reverseTarget,
lowWeightCorrection
)
);
amiPtr_().comm(coarseComm),
@ -819,7 +1024,8 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
{
const auto& AMI = amiPtr_();
const auto oldWarnComm = UPstream::commWarn(AMI.comm());
const auto oldWarnComm = UPstream::warnComm;
UPstream::warnComm = AMI.comm();
const label myRank = UPstream::myProcNo(AMI.comm());
Pout<< "PROCAGGLOMERATED :"
@ -838,7 +1044,7 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
const auto& weights = AMI.srcWeights();
labelList globalIDs(identity(nbrSize));
if (AMI.distributed())
if (AMI.distributed() && AMI.comm() != -1)
{
const auto& map = AMI.tgtMap();
forAll(map.subMap(), proci)
@ -877,6 +1083,7 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
<< UIndirectList<label>(globalIDs, addresses[facei])
<< endl;
}
UPstream::commWarn(oldWarnComm);
}
// From from owner to nbr side
{
@ -887,7 +1094,7 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
const auto& weights = AMI.tgtWeights();
labelList globalIDs(identity(this->size()));
if (AMI.distributed())
if (AMI.distributed() && AMI.comm() != -1)
{
const auto& map = AMI.srcMap();
forAll(map.subMap(), proci)
@ -928,8 +1135,7 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
}
}
Pout<< "DONE PROCAGGLOMERATED" << endl;
UPstream::commWarn(oldWarnComm);
}
UPstream::warnComm = oldWarnComm;
}
}
@ -974,6 +1180,15 @@ void Foam::cyclicAMIGAMGInterface::write(Ostream& os) const
{
os << token::SPACE;
AMI().writeData(os);
// Write processors in communicator
const label comm = AMI().comm();
if (comm != -1)
{
os << token::SPACE
<< UPstream::myProcNo(comm);
}
}
}

11
src/meshTools/AMIInterpolation/GAMG/interfaces/cyclicAMIGAMGInterface/cyclicAMIGAMGInterface.H
View File

@ -68,6 +68,11 @@ class cyclicAMIGAMGInterface
//- AMI interface
autoPtr<AMIPatchToPatchInterpolation> amiPtr_;
//- rank in AMI with respect to parent communicator.
//- Used for combining contributions from different processors if the
//- destination processor was not in the original set.
label myProcNo_;
// Private Member Functions
@ -213,6 +218,12 @@ public:
return reverseT_;
}
//- -1 or old local rank
virtual label myProcNo() const
{
return myProcNo_;
}
// I/O

16
src/meshTools/AMIInterpolation/patches/cyclicAMI/cyclicAMIPolyPatch/cyclicAMIPolyPatchTemplates.C
View File

@ -183,7 +183,7 @@ void Foam::cyclicAMIPolyPatch::initInterpolateUntransformed
{
const auto& AMI = (owner() ? this->AMI() : neighbPatch().AMI());
if (AMI.distributed())
if (AMI.distributed() && AMI.comm() != -1)
{
const auto& map = (owner() ? AMI.tgtMap() : AMI.srcMap());
@ -213,7 +213,7 @@ void Foam::cyclicAMIPolyPatch::initInterpolate
{
const auto& AMI = (owner() ? this->AMI() : neighbPatch().AMI());
if (!AMI.distributed())
if (!AMI.distributed() || AMI.comm() == -1)
{
return;
}
@ -273,12 +273,20 @@ Foam::tmp<Foam::Field<Type>> Foam::cyclicAMIPolyPatch::interpolate
const UList<Type>& defaultValues
) const
{
auto tresult = tmp<Field<Type>>::New(this->size(), Zero);
const auto& AMI = (owner() ? this->AMI() : neighbPatch().AMI());
const auto& map = (owner() ? AMI.tgtMap() : AMI.srcMap());
Field<Type> work;
if (AMI.distributed())
{
if (AMI.comm() == -1)
{
return tresult;
}
const auto& map = (owner() ? AMI.tgtMap() : AMI.srcMap());
// Receive (= copy) data from buffers into work. TBD: receive directly
// into slices of work.
map.receive
@ -291,8 +299,6 @@ Foam::tmp<Foam::Field<Type>> Foam::cyclicAMIPolyPatch::interpolate
}
const Field<Type>& fld = (AMI.distributed() ? work : localFld);
auto tresult = tmp<Field<Type>>::New(this->size(), Zero);
// Rotate fields (vector and non-spherical tensors)
constexpr bool transform_supported = is_rotational_vectorspace_v<Type>;

2
src/meshTools/AMIInterpolation/patches/cyclicAMI/cyclicAMIPolyPatch/cyclicAMIPolyPatchTopologyChange.C
View File

@ -312,7 +312,7 @@ void Foam::cyclicAMIPolyPatch::setAMIFaces()
autoPtr<mapDistribute> srcToTgtMap1;
autoPtr<mapDistribute> tgtToSrcMap1;
if (AMIPtr_->distributed())
if (AMIPtr_->distributed() && AMIPtr_().comm() != -1)
{
// Parallel running

4
src/sampling/meshToMesh/meshToMeshTemplates.C
View File

@ -537,7 +537,7 @@ void Foam::meshToMesh::mapSrcToTgt
(
AMIList[i].singlePatchProc(),
(
AMIList[i].distributed()
(AMIList[i].distributed() && AMIList[i].comm() != -1)
? AMIList[i].hasSrcMap() // pointer to map
: nullptr
),
@ -769,7 +769,7 @@ void Foam::meshToMesh::mapTgtToSrc
(
AMIList[i].singlePatchProc(),
(
AMIList[i].distributed()
(AMIList[i].distributed() && AMIList[i].comm() != -1)
? AMIList[i].hasTgtMap() // pointer to map
: nullptr
),