zhyang-dev/openfoam
1
0
Fork 0
mirror of https://develop.openfoam.com/Development/openfoam.git synced 2025-11-28 03:28:01 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

ENH: AMIInterpolation: use local communicator

Browse Source 
- allocate communicator with only those ranks that have
patch faces
- use this communicator everywhere
- communicator preserved during mesh motion
This commit is contained in:
mattijs
2024-12-05 16:34:54 +00:00
parent f276016896
commit 34c933c1ce
25 changed files with 1884 additions and 1079 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
etc/controlDict
View File

@ -240,6 +240,12 @@ OptimisationSwitches
// from all the individual wall patches. Set to 0/false to revert to // from all the individual wall patches. Set to 0/false to revert to
// <v2412 behaviour // <v2412 behaviour
//useCombinedWallPatch 0; //useCombinedWallPatch 0;
//- Use local communicator for AMI communication. Default for v2506.
//localAMIComm = 0; // old behaviour : all ranks included
//localAMIComm = 1; // (default) only ranks that have patch faces
//localAMIComm = 2; // like 1 but always include rank 0 so messages
// still come from processor0
} }

4
src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C
View File

@ -328,7 +328,7 @@ void Foam::GAMGAgglomeration::agglomerateLduAddressing
tmp<labelField> restrictMapInternalField; tmp<labelField> restrictMapInternalField;
// The finest mesh uses patchAddr from the original lduAdressing. // The finest mesh uses patchAddr from the original lduAdressing.
// the coarser levels create thei own adressing for faceCells // the coarser levels create their own adressing for faceCells
if (fineLevelIndex == 0) if (fineLevelIndex == 0)
{ {
restrictMapInternalField = restrictMapInternalField =
@ -466,7 +466,7 @@ void Foam::GAMGAgglomeration::procAgglomerateLduAddressing
procBoundaryFaceMap_.set(levelIndex, new labelListListList(0)); procBoundaryFaceMap_.set(levelIndex, new labelListListList(0));
// Collect meshes // Collect meshes. Does no renumbering
PtrList<lduPrimitiveMesh> otherMeshes; PtrList<lduPrimitiveMesh> otherMeshes;
lduPrimitiveMesh::gather(comm, myMesh, otherMeshes); lduPrimitiveMesh::gather(comm, myMesh, otherMeshes);

4
src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/masterCoarsestGAMGProcAgglomeration/masterCoarsestGAMGProcAgglomeration.C
View File

@ -182,8 +182,8 @@ bool Foam::masterCoarsestGAMGProcAgglomeration::agglomerate()
for (const auto& p : masterToProcs) for (const auto& p : masterToProcs)
{ {
Info<< '\t' << p[0] Info<< '\t' << p[0]
<< '\t' << p.size() << "\t\t" << p.size()
<< '\t' << "\t\t"
<< flatOutput(SubList<label>(p, p.size()-1, 1)) << flatOutput(SubList<label>(p, p.size()-1, 1))
<< endl; << endl;
} }

266
src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeBase.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2015-2017 OpenFOAM Foundation Copyright (C) 2015-2017 OpenFOAM Foundation
Copyright (C) 2015-2024 OpenCFD Ltd. Copyright (C) 2015-2025 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
@ -1001,17 +1001,39 @@ Foam::mapDistributeBase::mapDistributeBase
constructSize_(0), constructSize_(0),
subHasFlip_(false), subHasFlip_(false),
constructHasFlip_(false), constructHasFlip_(false),
comm_(-1), comm_(newComm),
schedulePtr_(nullptr) schedulePtr_(nullptr)
{ {
if (maps.empty()) // localRanks : gives for every map the index (=rank in original
// communicator) in subMap,constructMap that refers to
// the local data. -1 if the map did not contain any data
// (can happen for e.g. agglomerating of cyclicAMI data)
// newToOldRanks : gives for every rank in the newComm the ranks in the
// maps that agglomerate to it. This is used to decide
// - data was local and stays local
// - data that was remote and now becomes local
// - and same for remote
// Currently can contain -1 entries. Probably should be
// filtered...
// Find set index
label validMapi = -1;
forAll(maps, mapi)
{
if (maps.set(mapi) && localRanks[mapi] != -1)
{
validMapi = mapi;
break;
}
}
if (validMapi == -1)
{ {
return; return;
} }
comm_ = newComm; subHasFlip_ = maps[validMapi].subHasFlip();
subHasFlip_ = maps[0].subHasFlip(); constructHasFlip_ = maps[validMapi].constructHasFlip();
constructHasFlip_ = maps[0].constructHasFlip();
const label nNewRanks = newToOldRanks.size(); const label nNewRanks = newToOldRanks.size();
const label myNewRank = UPstream::myProcNo(newComm); const label myNewRank = UPstream::myProcNo(newComm);
@ -1031,15 +1053,20 @@ Foam::mapDistributeBase::mapDistributeBase
} }
// Sanity checks // Sanity checks
const auto& map0 = maps[0]; const auto& map0 = maps[validMapi];
forAll(maps, mapi) forAll(maps, mapi)
{ {
if (!maps.set(mapi) || localRanks[mapi] == -1)
{
continue;
}
const auto& map = maps[mapi]; const auto& map = maps[mapi];
if if
( (
(map.comm() != map0.comm()) //(map.comm() != map0.comm())
|| (map.subHasFlip() != map0.subHasFlip()) (map.subHasFlip() != map0.subHasFlip())
|| (map.constructHasFlip() != map0.constructHasFlip()) || (map.constructHasFlip() != map0.constructHasFlip())
) )
{ {
@ -1050,6 +1077,9 @@ Foam::mapDistributeBase::mapDistributeBase
<< " subHasFlip:" << map.subHasFlip() << " subHasFlip:" << map.subHasFlip()
<< " constructHasFlip:" << map.constructHasFlip() << " constructHasFlip:" << map.constructHasFlip()
<< " which is different from map 0 " << " which is different from map 0 "
<< " comm:" << map0.comm()
<< " subHasFlip:" << map0.subHasFlip()
<< " constructHasFlip:" << map0.constructHasFlip()
<< exit(FatalError); << exit(FatalError);
} }
@ -1069,6 +1099,38 @@ Foam::mapDistributeBase::mapDistributeBase
} }
// Determine start of constructed remote data. This is used to get the
// local offset which can then be used to get the relative subMap location.
labelListList startOfRemote(maps.size());
forAll(maps, mapi)
{
if (!maps.set(mapi))
{
continue;
}
const label nOldRanks = maps[mapi].constructMap().size();
labelList& starts = startOfRemote[mapi];
starts.setSize(nOldRanks, labelMax);
forAll(maps[mapi].constructMap(), oldRanki)
{
const labelList& map = maps[mapi].constructMap()[oldRanki];
forAll(map, i)
{
const label index
(
constructHasFlip_
? mag(map[i])-1
: map[i]
);
starts[oldRanki] = min(starts[oldRanki], index);
}
}
}
constructMap_.resize_nocopy(nNewRanks); constructMap_.resize_nocopy(nNewRanks);
subMap_.resize_nocopy(nNewRanks); subMap_.resize_nocopy(nNewRanks);
@ -1081,7 +1143,14 @@ Foam::mapDistributeBase::mapDistributeBase
forAll(maps, mapi) forAll(maps, mapi)
{ {
startOfLocal[mapi] = constructi; startOfLocal[mapi] = constructi;
const label localRank = localRanks[mapi]; const label localRank = localRanks[mapi];
if (!maps.set(mapi) || localRank == -1)
{
continue;
}
const auto& map = maps[mapi].constructMap()[localRank]; const auto& map = maps[mapi].constructMap()[localRank];
// Presize compaction array // Presize compaction array
@ -1093,31 +1162,6 @@ Foam::mapDistributeBase::mapDistributeBase
startOfLocal.last() = constructi; startOfLocal.last() = constructi;
// Determine start of constructed remote data. This is used to get the
// local offset which can then be used to get the relative subMap location.
labelListList startOfRemote(maps.size());
forAll(maps, mapi)
{
const label nOldProcs = maps[mapi].constructMap().size();
labelList& starts = startOfRemote[mapi];
starts.setSize(nOldProcs, labelMax);
forAll(maps[mapi].constructMap(), oldProci)
{
const labelList& map = maps[mapi].constructMap()[oldProci];
forAll(map, i)
{
const label index
(
constructHasFlip_
? mag(map[i])-1
: map[i]
);
starts[oldProci] = min(starts[oldProci], index);
}
}
}
// Construct map // Construct map
// ~~~~~~~~~~~~~ // ~~~~~~~~~~~~~
@ -1141,6 +1185,12 @@ Foam::mapDistributeBase::mapDistributeBase
forAll(maps, mapi) forAll(maps, mapi)
{ {
const label localRank = localRanks[mapi]; const label localRank = localRanks[mapi];
if (!maps.set(mapi) || localRank == -1)
{
continue;
}
const auto& map = maps[mapi].constructMap()[localRank]; const auto& map = maps[mapi].constructMap()[localRank];
const label offset = startOfLocal[mapi]; const label offset = startOfLocal[mapi];
@ -1166,22 +1216,68 @@ Foam::mapDistributeBase::mapDistributeBase
} }
} }
} }
if (constructi != startOfLocal.last())
{
FatalErrorInFunction
<< "constructi:" << constructi
<< " startOfLocal:" << startOfLocal.last()
<< exit(FatalError);
} }
}
// Filter remote construct data // Filter remote construct data
{ {
// Remote ranks that are now local // Remote ranks that are now local
// - store new index for mapping stencils // - store new index for mapping stencils
// - no need to construct since already // - no need to construct since already
const auto& oldProcs = newToOldRanks[myNewRank]; const auto& oldRanks = newToOldRanks[myNewRank];
// Determine number of old ranks. Note: should have passed in old-to-new
// ranks instead of new-to-old ranks?
label maxOldRank = -1;
for (const auto& elems : newToOldRanks)
{
for (const label el : elems)
{
maxOldRank = max(maxOldRank, el);
}
}
// Construct mapping from oldRanks to map. Note: could probably
// use some ListOps invert routine ...
labelList oldRankToMap(maxOldRank+1, -1);
{
forAll(localRanks, mapi)
{
const label localRank = localRanks[mapi];
if (localRank != -1)
{
oldRankToMap[localRank] = mapi;
}
}
}
forAll(maps, mapi) forAll(maps, mapi)
{ {
for (const label oldProci : oldProcs) if (!maps.set(mapi) || localRanks[mapi] == -1)
{ {
if (oldProci != localRanks[mapi]) continue;
}
for (const label oldRanki : oldRanks)
{ {
const auto& map = maps[mapi].constructMap()[oldProci]; if (oldRanki == -1)
{
continue;
}
if (oldRanki != localRanks[mapi])
{
// Note:
// - constructMap is sized with the map's communicator
const auto& map = maps[mapi].constructMap()[oldRanki];
if (!map.size()) if (!map.size())
{ {
@ -1191,11 +1287,11 @@ Foam::mapDistributeBase::mapDistributeBase
// The slots come from a local map so we can look up the // The slots come from a local map so we can look up the
// new location // new location
const label sourceMapi = localRanks.find(oldProci); const label sourceMapi = oldRankToMap[oldRanki];
const auto& subMap = const auto& subMap =
maps[sourceMapi].subMap()[localRanks[mapi]]; maps[sourceMapi].subMap()[localRanks[mapi]];
//Pout<< "From oldRank:" << oldProci //Pout<< "From oldRank:" << oldRanki
// << " sending to masterRank:" << localRanks[mapi] // << " sending to masterRank:" << localRanks[mapi]
// << " elements:" << flatOutput(subMap) // << " elements:" << flatOutput(subMap)
// << nl // << nl
@ -1205,7 +1301,7 @@ Foam::mapDistributeBase::mapDistributeBase
if (map.size() != subMap.size()) if (map.size() != subMap.size())
{ {
FatalErrorInFunction << "Problem:" FatalErrorInFunction << "Problem:"
<< "oldProci:" << oldProci << "oldRanki:" << oldRanki
<< " mapi:" << mapi << " mapi:" << mapi
<< " constructMap:" << map.size() << " constructMap:" << map.size()
<< " sourceMapi:" << sourceMapi << " sourceMapi:" << sourceMapi
@ -1215,7 +1311,7 @@ Foam::mapDistributeBase::mapDistributeBase
const label offset = startOfLocal[sourceMapi]; const label offset = startOfLocal[sourceMapi];
// Construct map starts after the local data // Construct map starts after the local data
const label nMapLocal = startOfRemote[mapi][oldProci]; const label nMapLocal = startOfRemote[mapi][oldRanki];
auto& cptMap = compactMaps[mapi]; auto& cptMap = compactMaps[mapi];
forAll(map, i) forAll(map, i)
@ -1238,7 +1334,7 @@ Foam::mapDistributeBase::mapDistributeBase
) )
{ {
FatalErrorInFunction<< "Duplicate insertion" FatalErrorInFunction<< "Duplicate insertion"
<< "From oldProc:" << oldProci << "From oldRank:" << oldRanki
<< " on map:" << mapi << " on map:" << mapi
<< " at index:" << i << " at index:" << i
<< " have construct slot:" << index << " have construct slot:" << index
@ -1260,32 +1356,51 @@ Foam::mapDistributeBase::mapDistributeBase
// Either loop over all old ranks and filter out ones already handled // Either loop over all old ranks and filter out ones already handled
// or loop over all new ranks and avoid myNewRank // or loop over all new ranks and avoid myNewRank
forAll(newToOldRanks, newProci) forAll(newToOldRanks, newRanki)
{ {
if (newProci != myNewRank) if (newRanki != myNewRank)
{ {
const auto& oldProcs = newToOldRanks[newProci]; const auto& oldRanks = newToOldRanks[newRanki];
label allSize = 0; label allSize = 0;
forAll(maps, mapi) forAll(maps, mapi)
{ {
for (const label oldProci : oldProcs) if (!maps.set(mapi))
{ {
allSize += maps[mapi].constructMap()[oldProci].size(); continue;
}
for (const label oldRanki : oldRanks)
{
if (oldRanki == -1)
{
continue;
}
allSize += maps[mapi].constructMap()[oldRanki].size();
} }
} }
labelList& myConstruct = constructMap_[newProci]; labelList& myConstruct = constructMap_[newRanki];
myConstruct.resize_nocopy(allSize); myConstruct.resize_nocopy(allSize);
allSize = 0; allSize = 0;
forAll(maps, mapi) forAll(maps, mapi)
{ {
for (const label oldProci : oldProcs) if (!maps.set(mapi))
{ {
const auto& map = maps[mapi].constructMap()[oldProci]; continue;
}
for (const label oldRanki : oldRanks)
{
if (oldRanki == -1)
{
continue;
}
const auto& map = maps[mapi].constructMap()[oldRanki];
// Construct map starts after the local data // Construct map starts after the local data
const label nMapLocal = startOfRemote[mapi][oldProci]; const label nMapLocal = startOfRemote[mapi][oldRanki];
SubList<label> slice(myConstruct, map.size(), allSize); SubList<label> slice(myConstruct, map.size(), allSize);
if (constructHasFlip_) if (constructHasFlip_)
@ -1340,6 +1455,12 @@ Foam::mapDistributeBase::mapDistributeBase
forAll(maps, mapi) forAll(maps, mapi)
{ {
const label localRank = localRanks[mapi]; const label localRank = localRanks[mapi];
if (!maps.set(mapi) || localRank == -1)
{
continue;
}
allSize += maps[mapi].subMap()[localRank].size(); allSize += maps[mapi].subMap()[localRank].size();
} }
@ -1349,6 +1470,12 @@ Foam::mapDistributeBase::mapDistributeBase
forAll(maps, mapi) forAll(maps, mapi)
{ {
const label localRank = localRanks[mapi]; const label localRank = localRanks[mapi];
if (!maps.set(mapi) || localRank == -1)
{
continue;
}
const auto& map = maps[mapi].subMap()[localRank]; const auto& map = maps[mapi].subMap()[localRank];
SubList<label> slice(mySub, map.size(), allSize); SubList<label> slice(mySub, map.size(), allSize);
@ -1377,30 +1504,49 @@ Foam::mapDistributeBase::mapDistributeBase
} }
} }
// Filter remote sub data // Filter remote sub data
forAll(newToOldRanks, newProci) forAll(newToOldRanks, newRanki)
{ {
if (newProci != myNewRank) if (newRanki != myNewRank)
{ {
const auto& oldProcs = newToOldRanks[newProci]; const auto& oldRanks = newToOldRanks[newRanki];
label allSize = 0; label allSize = 0;
forAll(maps, mapi) forAll(maps, mapi)
{ {
for (const label oldProci : oldProcs) if (!maps.set(mapi))
{ {
allSize += maps[mapi].subMap()[oldProci].size(); continue;
}
for (const label oldRanki : oldRanks)
{
if (oldRanki == -1)
{
continue;
}
allSize += maps[mapi].subMap()[oldRanki].size();
} }
} }
labelList& mySub = subMap_[newProci]; labelList& mySub = subMap_[newRanki];
mySub.resize_nocopy(allSize); mySub.resize_nocopy(allSize);
allSize = 0; allSize = 0;
for (const label oldProci : oldProcs) for (const label oldRanki : oldRanks)
{ {
if (oldRanki == -1)
{
continue;
}
forAll(maps, mapi) forAll(maps, mapi)
{ {
const auto& map = maps[mapi].subMap()[oldProci]; if (!maps.set(mapi))
{
continue;
}
const auto& map = maps[mapi].subMap()[oldRanki];
SubList<label> slice(mySub, map.size(), allSize); SubList<label> slice(mySub, map.size(), allSize);
if (subHasFlip_) if (subHasFlip_)
{ {

22
src/finiteVolume/fields/fvPatchFields/constraint/cyclicACMI/cyclicACMIFvPatchField.C
View File

@ -345,7 +345,9 @@ template<class Type>
Foam::tmp<Foam::Field<Type>> Foam::tmp<Foam::Field<Type>>
Foam::cyclicACMIFvPatchField<Type>::patchNeighbourField() const Foam::cyclicACMIFvPatchField<Type>::patchNeighbourField() const
{ {
if (this->ownerAMI().distributed() && cacheNeighbourField()) const auto& AMI = this->ownerAMI();
if (AMI.distributed() && cacheNeighbourField() && AMI.comm() != -1)
{ {
if (!this->ready()) if (!this->ready())
{ {
@ -441,7 +443,9 @@ void Foam::cyclicACMIFvPatchField<Type>::initEvaluate
this->updateCoeffs(); this->updateCoeffs();
} }
if (this->ownerAMI().distributed() && cacheNeighbourField()) const auto& AMI = this->ownerAMI();
if (AMI.distributed() && cacheNeighbourField() && AMI.comm() != -1)
{ {
if (commsType != UPstream::commsTypes::nonBlocking) if (commsType != UPstream::commsTypes::nonBlocking)
{ {
@ -502,7 +506,7 @@ void Foam::cyclicACMIFvPatchField<Type>::evaluate
const auto& AMI = this->ownerAMI(); const auto& AMI = this->ownerAMI();
if (AMI.distributed() && cacheNeighbourField()) if (AMI.distributed() && cacheNeighbourField() && AMI.comm() != -1)
{ {
// Calculate patchNeighbourField // Calculate patchNeighbourField
if (commsType != UPstream::commsTypes::nonBlocking) if (commsType != UPstream::commsTypes::nonBlocking)
@ -564,7 +568,9 @@ void Foam::cyclicACMIFvPatchField<Type>::initInterfaceMatrixUpdate
const Pstream::commsTypes commsType const Pstream::commsTypes commsType
) const ) const
{ {
if (this->ownerAMI().distributed()) const auto& AMI = this->ownerAMI();
if (AMI.distributed() && AMI.comm() != -1)
{ {
// Start sending // Start sending
if (commsType != UPstream::commsTypes::nonBlocking) if (commsType != UPstream::commsTypes::nonBlocking)
@ -640,7 +646,9 @@ void Foam::cyclicACMIFvPatchField<Type>::updateInterfaceMatrix
solveScalarField pnf; solveScalarField pnf;
if (this->ownerAMI().distributed()) const auto& AMI = this->ownerAMI();
if (AMI.distributed() && AMI.comm() != -1)
{ {
if (commsType != UPstream::commsTypes::nonBlocking) if (commsType != UPstream::commsTypes::nonBlocking)
{ {
@ -700,7 +708,7 @@ void Foam::cyclicACMIFvPatchField<Type>::initInterfaceMatrixUpdate
{ {
const auto& AMI = this->ownerAMI(); const auto& AMI = this->ownerAMI();
if (AMI.distributed()) if (AMI.distributed() && AMI.comm() != -1)
{ {
if (commsType != UPstream::commsTypes::nonBlocking) if (commsType != UPstream::commsTypes::nonBlocking)
{ {
@ -764,7 +772,7 @@ void Foam::cyclicACMIFvPatchField<Type>::updateInterfaceMatrix
Field<Type> pnf; Field<Type> pnf;
if (AMI.distributed()) if (AMI.distributed() && AMI.comm() != -1)
{ {
if (commsType != UPstream::commsTypes::nonBlocking) if (commsType != UPstream::commsTypes::nonBlocking)
{ {

20
src/finiteVolume/fields/fvPatchFields/constraint/cyclicAMI/cyclicAMIFvPatchField.C
View File

@ -338,7 +338,9 @@ template<class Type>
Foam::tmp<Foam::Field<Type>> Foam::tmp<Foam::Field<Type>>
Foam::cyclicAMIFvPatchField<Type>::patchNeighbourField() const Foam::cyclicAMIFvPatchField<Type>::patchNeighbourField() const
{ {
if (this->ownerAMI().distributed() && cacheNeighbourField()) const auto& AMI = this->ownerAMI();
if (AMI.distributed() && cacheNeighbourField() && AMI.comm() != -1)
{ {
if (!this->ready()) if (!this->ready())
{ {
@ -442,7 +444,9 @@ void Foam::cyclicAMIFvPatchField<Type>::initEvaluate
this->updateCoeffs(); this->updateCoeffs();
} }
if (this->ownerAMI().distributed() && cacheNeighbourField()) const auto& AMI = this->ownerAMI();
if (AMI.distributed() && cacheNeighbourField() && AMI.comm() != -1)
{ {
//DebugPout //DebugPout
// << "*** cyclicAMIFvPatchField::initEvaluate() :" // << "*** cyclicAMIFvPatchField::initEvaluate() :"
@ -506,7 +510,7 @@ void Foam::cyclicAMIFvPatchField<Type>::evaluate
const auto& AMI = this->ownerAMI(); const auto& AMI = this->ownerAMI();
if (AMI.distributed() && cacheNeighbourField()) if (AMI.distributed() && cacheNeighbourField() && AMI.comm() != -1)
{ {
// Calculate patchNeighbourField // Calculate patchNeighbourField
if (commsType != UPstream::commsTypes::nonBlocking) if (commsType != UPstream::commsTypes::nonBlocking)
@ -574,7 +578,9 @@ void Foam::cyclicAMIFvPatchField<Type>::initInterfaceMatrixUpdate
const Pstream::commsTypes commsType const Pstream::commsTypes commsType
) const ) const
{ {
if (this->ownerAMI().distributed()) const auto& AMI = this->ownerAMI();
if (AMI.distributed() && AMI.comm() != -1)
{ {
// Start sending // Start sending
if (commsType != UPstream::commsTypes::nonBlocking) if (commsType != UPstream::commsTypes::nonBlocking)
@ -645,7 +651,7 @@ void Foam::cyclicAMIFvPatchField<Type>::updateInterfaceMatrix
solveScalarField pnf; solveScalarField pnf;
if (AMI.distributed()) if (AMI.distributed() && AMI.comm() != -1)
{ {
if (commsType != UPstream::commsTypes::nonBlocking) if (commsType != UPstream::commsTypes::nonBlocking)
{ {
@ -714,7 +720,7 @@ void Foam::cyclicAMIFvPatchField<Type>::initInterfaceMatrixUpdate
{ {
const auto& AMI = this->ownerAMI(); const auto& AMI = this->ownerAMI();
if (AMI.distributed()) if (AMI.distributed() && AMI.comm() != -1)
{ {
if (commsType != UPstream::commsTypes::nonBlocking) if (commsType != UPstream::commsTypes::nonBlocking)
{ {
@ -783,7 +789,7 @@ void Foam::cyclicAMIFvPatchField<Type>::updateInterfaceMatrix
Field<Type> pnf; Field<Type> pnf;
if (AMI.distributed()) if (AMI.distributed() && AMI.comm() != -1)
{ {
if (commsType != UPstream::commsTypes::nonBlocking) if (commsType != UPstream::commsTypes::nonBlocking)
{ {

2
src/finiteVolume/fvMesh/fvPatches/constraint/cyclicAMI/cyclicAMIFvPatch.C
View File

@ -289,7 +289,7 @@ void Foam::cyclicAMIFvPatch::movePoints()
} }
scalarField srcMeshPhi(phip); scalarField srcMeshPhi(phip);
if (AMI().distributed()) if (AMI().distributed() && AMI().comm() != -1)
{ {
AMI().srcMap().distribute(srcMeshPhi); AMI().srcMap().distribute(srcMeshPhi);
} }

186
src/meshTools/AMIInterpolation/AMIInterpolation/AMIInterpolation.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2015-2023 OpenCFD Ltd. Copyright (C) 2015-2024 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
@ -33,6 +33,7 @@ License
#include "profiling.H" #include "profiling.H"
#include "triangle.H" #include "triangle.H"
#include "OFstream.H" #include "OFstream.H"
#include "registerSwitch.H"
#include <numeric> // For std::iota #include <numeric> // For std::iota
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * // // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
@ -46,6 +47,17 @@ namespace Foam
bool Foam::AMIInterpolation::cacheIntersections_ = false; bool Foam::AMIInterpolation::cacheIntersections_ = false;
int Foam::AMIInterpolation::localComm_
(
debug::optimisationSwitch("localAMIComm", 1)
);
registerOptSwitch
(
"localAMIComm",
int,
Foam::AMIInterpolation::localComm_
);
// * * * * * * * * * * * * Protected Member Functions * * * * * * * * * * * // // * * * * * * * * * * * * Protected Member Functions * * * * * * * * * * * //
@ -77,7 +89,9 @@ Foam::AMIInterpolation::createTree
Foam::label Foam::AMIInterpolation::calcDistribution Foam::label Foam::AMIInterpolation::calcDistribution
( (
const primitivePatch& srcPatch, const primitivePatch& srcPatch,
const primitivePatch& tgtPatch const primitivePatch& tgtPatch,
const label comm,
autoPtr<UPstream::communicator>& geomComm
) const ) const
{ {
// Either not parallel or no faces on any processor // Either not parallel or no faces on any processor
@ -85,12 +99,40 @@ Foam::label Foam::AMIInterpolation::calcDistribution
if (Pstream::parRun()) if (Pstream::parRun())
{ {
bool hasLocalFaces = false;
if (localComm_ == 0)
{
// Backwards compatible : all processors involved
hasLocalFaces = true;
}
else if (localComm_ == 1)
{
// Only if locally have faces
hasLocalFaces = (srcPatch.size() > 0 || tgtPatch.size() > 0);
}
else
{
// If master (so messages always come from master) or if locally
// have faces
hasLocalFaces =
(
UPstream::master(comm)
|| srcPatch.size() > 0
|| tgtPatch.size() > 0
);
}
// Better to use MPI_Comm_split? So only provide local information
// (hasLocalFaces). Problem is that we don't know who else is in the
// set. Whereas now, because we all loop over the same data in the same
// order we coud find out (except we don't use this information?)
const bitSet hasFaces const bitSet hasFaces
( (
UPstream::listGatherValues<bool> UPstream::allGatherValues<bool>
( (
(srcPatch.size() > 0 || tgtPatch.size() > 0), hasLocalFaces,
comm_ comm
) )
); );
@ -98,18 +140,48 @@ Foam::label Foam::AMIInterpolation::calcDistribution
if (nHaveFaces == 1) if (nHaveFaces == 1)
{ {
// Release any previously allocated communicator
geomComm.clear();
proci = hasFaces.find_first(); proci = hasFaces.find_first();
DebugInFunction DebugInFunction
<< "AMI local to processor" << proci << endl; << "AMI local to processor" << proci << endl;
} }
else if (nHaveFaces > 1) else if (nHaveFaces > 1)
{ {
proci = -1; if (hasLocalFaces)
DebugInFunction {
<< "AMI split across multiple processors" << endl; geomComm.reset
(
new UPstream::communicator
(
comm,
hasFaces.sortedToc()
)
);
if (debug)
{
Pout<< "Allocated geomComm:" << geomComm()
<< " from " << nHaveFaces
<< " processors out of " << UPstream::nProcs(comm)
<< endl;
}
}
else
{
geomComm.reset(new UPstream::communicator());
if (debug & 2)
{
Pout<< "Allocated dummy geomComm:" << geomComm()
<< " since no src:" << srcPatch.size()
<< " and no tgt:" << tgtPatch.size() << endl;
}
} }
Pstream::broadcast(proci, comm_); proci = -1;
DebugInFunction
<< "AMI split across multiple processors "
<< flatOutput(hasFaces.sortedToc()) << endl;
}
} }
return proci; return proci;
@ -207,7 +279,7 @@ void Foam::AMIInterpolation::normaliseWeights
} }
} }
if (output) if (output && comm != -1)
{ {
// Note: change global communicator since gMin,gAverage etc don't // Note: change global communicator since gMin,gAverage etc don't
// support user communicator // support user communicator
@ -216,7 +288,8 @@ void Foam::AMIInterpolation::normaliseWeights
if (returnReduceOr(wght.size())) if (returnReduceOr(wght.size()))
{ {
Info<< indent Info.masterStream(comm)
<< indent
<< "AMI: Patch " << patchName << "AMI: Patch " << patchName
<< " sum(weights)" << " sum(weights)"
<< " min:" << gMin(wghtSum) << " min:" << gMin(wghtSum)
@ -227,7 +300,8 @@ void Foam::AMIInterpolation::normaliseWeights
if (nLow) if (nLow)
{ {
Info<< indent Info.masterStream(comm)
<< indent
<< "AMI: Patch " << patchName << "AMI: Patch " << patchName
<< " identified " << nLow << " identified " << nLow
<< " faces with weights less than " << lowWeightTol << " faces with weights less than " << lowWeightTol
@ -260,14 +334,14 @@ void Foam::AMIInterpolation::agglomerate
{ {
addProfiling(ami, "AMIInterpolation::agglomerate"); addProfiling(ami, "AMIInterpolation::agglomerate");
label sourceCoarseSize = const label sourceCoarseSize =
( (
sourceRestrictAddressing.size() sourceRestrictAddressing.size()
? max(sourceRestrictAddressing)+1 ? max(sourceRestrictAddressing)+1
: 0 : 0
); );
label targetCoarseSize = const label targetCoarseSize =
( (
targetRestrictAddressing.size() targetRestrictAddressing.size()
? max(targetRestrictAddressing)+1 ? max(targetRestrictAddressing)+1
@ -289,8 +363,22 @@ void Foam::AMIInterpolation::agglomerate
// Agglomerate weights and indices // Agglomerate weights and indices
if (targetMapPtr) if (targetMapPtr)
{ {
// We are involved in the communicator but our maps are still empty.
// Fix 'm up so they are the same size as the communicator.
const mapDistribute& map = *targetMapPtr; const mapDistribute& map = *targetMapPtr;
if (map.constructMap().empty())
{
auto& cMap = const_cast<labelListList&>(map.constructMap());
cMap.resize_nocopy(UPstream::nProcs(map.comm()));
}
if (map.subMap().empty())
{
auto& cMap = const_cast<labelListList&>(map.subMap());
cMap.resize_nocopy(UPstream::nProcs(map.comm()));
}
// Get all restriction addressing. // Get all restriction addressing.
labelList allRestrict(targetRestrictAddressing); labelList allRestrict(targetRestrictAddressing);
map.distribute(allRestrict); map.distribute(allRestrict);
@ -621,7 +709,8 @@ Foam::AMIInterpolation::AMIInterpolation
reverseTarget_(fineAMI.reverseTarget_), reverseTarget_(fineAMI.reverseTarget_),
lowWeightCorrection_(-1.0), lowWeightCorrection_(-1.0),
singlePatchProc_(fineAMI.singlePatchProc_), singlePatchProc_(fineAMI.singlePatchProc_),
comm_(fineAMI.comm_), comm_(fineAMI.comm()), // use fineAMI geomComm if present, comm otherwise
geomComm_(),
srcMagSf_(), srcMagSf_(),
srcAddress_(), srcAddress_(),
srcWeights_(), srcWeights_(),
@ -681,6 +770,26 @@ Foam::AMIInterpolation::AMIInterpolation
// Agglomerate addresses and weights // Agglomerate addresses and weights
if (comm() != -1)
{
//Pout<< "** agglomerating srcAddress, tgtMap" << endl;
//if (fineAMI.tgtMapPtr_.valid())
//{
// const auto& fineTgtMap = fineAMI.tgtMapPtr_();
// Pout<< " fineAMI.tgtMapPtr_ comm:" << fineTgtMap.comm()
// << " procs:"
// << (
// fineTgtMap.comm() != -1
// ? UPstream::procID(fineTgtMap.comm())
// : labelList::null()
// )
// << endl;
//}
//else
//{
// Pout<< " NO fineAMI.tgtMapPtr_" << endl;
//}
//
agglomerate agglomerate
( (
fineAMI.tgtMapPtr_, fineAMI.tgtMapPtr_,
@ -696,9 +805,26 @@ Foam::AMIInterpolation::AMIInterpolation
srcWeights_, srcWeights_,
srcWeightsSum_, srcWeightsSum_,
tgtMapPtr_, tgtMapPtr_,
comm_ comm()
); );
//Pout<< "** agglomerating tgtAddress, srcMap" << endl;
//if (fineAMI.srcMapPtr_.valid())
//{
// const auto& fineSrcMap = fineAMI.srcMapPtr_();
// Pout<< " fineAMI.srcMapPtr_ comm:" << fineSrcMap.comm()
// << " procs:"
// << (
// fineSrcMap.comm() != -1
// ? UPstream::procID(fineSrcMap.comm())
// : labelList::null()
// )
// << endl;
//}
//else
//{
// Pout<< " NO fineAMI.srcMapPtr_" << endl;
//}
agglomerate agglomerate
( (
fineAMI.srcMapPtr_, fineAMI.srcMapPtr_,
@ -714,9 +840,10 @@ Foam::AMIInterpolation::AMIInterpolation
tgtWeights_, tgtWeights_,
tgtWeightsSum_, tgtWeightsSum_,
srcMapPtr_, srcMapPtr_,
comm_ comm()
); );
} }
}
Foam::AMIInterpolation::AMIInterpolation(const AMIInterpolation& ami) Foam::AMIInterpolation::AMIInterpolation(const AMIInterpolation& ami)
@ -726,6 +853,7 @@ Foam::AMIInterpolation::AMIInterpolation(const AMIInterpolation& ami)
lowWeightCorrection_(ami.lowWeightCorrection_), lowWeightCorrection_(ami.lowWeightCorrection_),
singlePatchProc_(ami.singlePatchProc_), singlePatchProc_(ami.singlePatchProc_),
comm_(ami.comm_), comm_(ami.comm_),
geomComm_(ami.geomComm_), // ? steals communicator
srcMagSf_(ami.srcMagSf_), srcMagSf_(ami.srcMagSf_),
srcAddress_(ami.srcAddress_), srcAddress_(ami.srcAddress_),
srcWeights_(ami.srcWeights_), srcWeights_(ami.srcWeights_),
@ -748,7 +876,7 @@ Foam::AMIInterpolation::AMIInterpolation(Istream& is)
reverseTarget_(readBool(is)), reverseTarget_(readBool(is)),
lowWeightCorrection_(readScalar(is)), lowWeightCorrection_(readScalar(is)),
singlePatchProc_(readLabel(is)), singlePatchProc_(readLabel(is)),
comm_(readLabel(is)), comm_(readLabel(is)), // either geomComm_ or comm_ from sending side
srcMagSf_(is), srcMagSf_(is),
srcAddress_(is), srcAddress_(is),
@ -768,7 +896,10 @@ Foam::AMIInterpolation::AMIInterpolation(Istream& is)
upToDate_(readBool(is)) upToDate_(readBool(is))
{ {
if (singlePatchProc_ == -1) // Hopefully no need to stream geomComm_ since only used in processor
// agglomeration?
if (singlePatchProc_ == -1 && comm_ != -1)
{ {
srcMapPtr_.reset(new mapDistribute(is)); srcMapPtr_.reset(new mapDistribute(is));
tgtMapPtr_.reset(new mapDistribute(is)); tgtMapPtr_.reset(new mapDistribute(is));
@ -816,6 +947,7 @@ bool Foam::AMIInterpolation::calculate
ttgtPatch0_.cref(tgtPatch); ttgtPatch0_.cref(tgtPatch);
} }
// Note: use original communicator for statistics
label srcTotalSize = returnReduce label srcTotalSize = returnReduce
( (
srcPatch.size(), srcPatch.size(),
@ -842,7 +974,11 @@ bool Foam::AMIInterpolation::calculate
return false; return false;
} }
singlePatchProc_ = calcDistribution(srcPatch, tgtPatch); // Calculate:
// - which processors have faces
// - allocates a communicator (geomComm_) for those
// - if it is only one processor that holds all faces
singlePatchProc_ = calcDistribution(srcPatch, tgtPatch, comm_, geomComm_);
Info<< indent << "AMI: Patch source faces: " << srcTotalSize << nl Info<< indent << "AMI: Patch source faces: " << srcTotalSize << nl
<< indent << "AMI: Patch target faces: " << tgtTotalSize << nl; << indent << "AMI: Patch target faces: " << tgtTotalSize << nl;
@ -913,7 +1049,7 @@ void Foam::AMIInterpolation::append
newPtr->calculate(srcPatch, tgtPatch); newPtr->calculate(srcPatch, tgtPatch);
// If parallel then combine the mapDistribution and re-index // If parallel then combine the mapDistribution and re-index
if (distributed()) if (distributed() && comm() != -1)
{ {
labelListList& srcSubMap = srcMapPtr_->subMap(); labelListList& srcSubMap = srcMapPtr_->subMap();
labelListList& srcConstructMap = srcMapPtr_->constructMap(); labelListList& srcConstructMap = srcMapPtr_->constructMap();
@ -1107,7 +1243,7 @@ void Foam::AMIInterpolation::normaliseWeights
conformal, conformal,
output, output,
lowWeightCorrection_, lowWeightCorrection_,
comm_ comm()
); );
normaliseWeights normaliseWeights
@ -1120,7 +1256,7 @@ void Foam::AMIInterpolation::normaliseWeights
conformal, conformal,
output, output,
lowWeightCorrection_, lowWeightCorrection_,
comm_ comm()
); );
} }
@ -1345,7 +1481,7 @@ bool Foam::AMIInterpolation::writeData(Ostream& os) const
<< token::SPACE<< reverseTarget() << token::SPACE<< reverseTarget()
<< token::SPACE<< lowWeightCorrection() << token::SPACE<< lowWeightCorrection()
<< token::SPACE<< singlePatchProc() << token::SPACE<< singlePatchProc()
<< token::SPACE<< comm() << token::SPACE<< comm() // either geomComm_ or comm_
<< token::SPACE<< srcMagSf() << token::SPACE<< srcMagSf()
<< token::SPACE<< srcAddress() << token::SPACE<< srcAddress()
@ -1361,7 +1497,7 @@ bool Foam::AMIInterpolation::writeData(Ostream& os) const
<< token::SPACE<< upToDate(); << token::SPACE<< upToDate();
if (distributed()) if (distributed() && comm() != -1)
{ {
os << token::SPACE<< srcMap() os << token::SPACE<< srcMap()
<< token::SPACE<< tgtMap(); << token::SPACE<< tgtMap();

19
src/meshTools/AMIInterpolation/AMIInterpolation/AMIInterpolation.H
View File

@ -6,7 +6,7 @@
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2016-2023 OpenCFD Ltd. Copyright (C) 2016-2024 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
@ -80,6 +80,11 @@ public:
static bool cacheIntersections_; static bool cacheIntersections_;
//- localComm : 0 : all processors (backwards compatible)
//- localComm : 1 : only processors with patch faces
//- localComm : 2 : like 1 but add master processor always
static int localComm_;
protected: protected:
@ -106,6 +111,10 @@ protected:
//- Communicator to use for parallel operations. //- Communicator to use for parallel operations.
label comm_; label comm_;
//- Communicator to use for geometry calculations. Not valid (-1) on
//- processors that do not have faces
autoPtr<UPstream::communicator> geomComm_;
// Source patch // Source patch
@ -181,11 +190,15 @@ protected:
const primitivePatch& patch const primitivePatch& patch
) const; ) const;
//- Calculate if patches are on multiple processors //- Calculate if patches are on multiple processors. Allocates
//- local communicator and returns -1 or processor containing all
//- faces
label calcDistribution label calcDistribution
( (
const primitivePatch& srcPatch, const primitivePatch& srcPatch,
const primitivePatch& tgtPatch const primitivePatch& tgtPatch,
const label comm,
autoPtr<UPstream::communicator>& geomComm
) const; ) const;
//- Project points to surface //- Project points to surface

7
src/meshTools/AMIInterpolation/AMIInterpolation/AMIInterpolationI.H
View File

@ -103,9 +103,16 @@ inline Foam::label Foam::AMIInterpolation::singlePatchProc() const noexcept
inline Foam::label Foam::AMIInterpolation::comm() const inline Foam::label Foam::AMIInterpolation::comm() const
{
if (geomComm_.valid())
{
return geomComm_();
}
else
{ {
return comm_; return comm_;
} }
}
inline Foam::label Foam::AMIInterpolation::comm(const label newComm) inline Foam::label Foam::AMIInterpolation::comm(const label newComm)

16
src/meshTools/AMIInterpolation/AMIInterpolation/AMIInterpolationTemplates.C
View File

@ -142,9 +142,15 @@ void Foam::AMIInterpolation::interpolateToTarget
result.setSize(tgtAddress_.size()); result.setSize(tgtAddress_.size());
List<Type> work; List<Type> work;
if (distributed()) if (distributed() && srcMapPtr_)
{ {
const mapDistribute& map = srcMapPtr_(); const mapDistribute& map = srcMapPtr_();
if (map.comm() == -1)
{
return;
}
work.resize_nocopy(map.constructSize()); work.resize_nocopy(map.constructSize());
SubList<Type>(work, fld.size()) = fld; // deep copy SubList<Type>(work, fld.size()) = fld; // deep copy
map.distribute(work); map.distribute(work);
@ -203,9 +209,15 @@ void Foam::AMIInterpolation::interpolateToSource
result.setSize(srcAddress_.size()); result.setSize(srcAddress_.size());
List<Type> work; List<Type> work;
if (distributed()) if (distributed() && tgtMapPtr_)
{ {
const mapDistribute& map = tgtMapPtr_(); const mapDistribute& map = tgtMapPtr_();
if (map.comm() == -1)
{
return;
}
work.resize_nocopy(map.constructSize()); work.resize_nocopy(map.constructSize());
SubList<Type>(work, fld.size()) = fld; // deep copy SubList<Type>(work, fld.size()) = fld; // deep copy
map.distribute(work); map.distribute(work);

16
src/meshTools/AMIInterpolation/AMIInterpolation/advancingFrontAMI/advancingFrontAMI.C
View File

@ -63,7 +63,7 @@ void Foam::advancingFrontAMI::checkPatches() const
} }
if (requireMatch_) if (requireMatch_ && comm() != -1)
{ {
const scalar maxBoundsError = 0.05; const scalar maxBoundsError = 0.05;
@ -74,14 +74,14 @@ void Foam::advancingFrontAMI::checkPatches() const
bbSrc.min(), bbSrc.min(),
minOp<point>(), minOp<point>(),
UPstream::msgType(), UPstream::msgType(),
comm_ comm()
); );
Foam::reduce Foam::reduce
( (
bbSrc.max(), bbSrc.max(),
maxOp<point>(), maxOp<point>(),
UPstream::msgType(), UPstream::msgType(),
comm_ comm()
); );
boundBox bbTgt(tgt.points(), tgt.meshPoints(), false); boundBox bbTgt(tgt.points(), tgt.meshPoints(), false);
Foam::reduce Foam::reduce
@ -89,14 +89,14 @@ void Foam::advancingFrontAMI::checkPatches() const
bbTgt.min(), bbTgt.min(),
minOp<point>(), minOp<point>(),
UPstream::msgType(), UPstream::msgType(),
comm_ comm()
); );
Foam::reduce Foam::reduce
( (
bbTgt.max(), bbTgt.max(),
maxOp<point>(), maxOp<point>(),
UPstream::msgType(), UPstream::msgType(),
comm_ comm()
); );
boundBox bbTgtInf(bbTgt); boundBox bbTgtInf(bbTgt);
@ -178,7 +178,7 @@ void Foam::advancingFrontAMI::createExtendedTgtPatch()
// Original faces from tgtPatch // Original faces from tgtPatch
// Note: in globalIndexing since might be remote // Note: in globalIndexing since might be remote
globalIndex globalTgtFaces(tgtPatch0().size(), comm_); globalIndex globalTgtFaces(tgtPatch0().size(), comm());
distributeAndMergePatches distributeAndMergePatches
( (
map, map,
@ -465,7 +465,7 @@ void Foam::advancingFrontAMI::triangulatePatch
void Foam::advancingFrontAMI::nonConformalCorrection() void Foam::advancingFrontAMI::nonConformalCorrection()
{ {
if (!requireMatch_ && distributed()) if (!requireMatch_ && distributed() && comm() != -1)
{ {
scalarList newTgtMagSf(std::move(tgtMagSf_)); scalarList newTgtMagSf(std::move(tgtMagSf_));
@ -587,7 +587,7 @@ bool Foam::advancingFrontAMI::calculate
{ {
// Create a representation of the target patch that covers the source // Create a representation of the target patch that covers the source
// patch // patch
if (distributed()) if (distributed() && comm() != -1)
{ {
createExtendedTgtPatch(); createExtendedTgtPatch();
} }

53
src/meshTools/AMIInterpolation/AMIInterpolation/advancingFrontAMI/advancingFrontAMIParallelOps.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2018-2022 OpenCFD Ltd. Copyright (C) 2018-2024 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
@ -74,13 +74,13 @@ void Foam::advancingFrontAMI::distributePatches
List<labelList>& faceIDs List<labelList>& faceIDs
) const ) const
{ {
faces.setSize(Pstream::nProcs(comm_)); faces.setSize(Pstream::nProcs(comm()));
points.setSize(Pstream::nProcs(comm_)); points.setSize(Pstream::nProcs(comm()));
faceIDs.setSize(Pstream::nProcs(comm_)); faceIDs.setSize(Pstream::nProcs(comm()));
PstreamBuffers pBufs(comm_); PstreamBuffers pBufs(comm());
for (const int domain : Pstream::allProcs(comm_)) for (const int domain : Pstream::allProcs(comm()))
{ {
const labelList& sendElems = map.subMap()[domain]; const labelList& sendElems = map.subMap()[domain];
@ -103,13 +103,13 @@ void Foam::advancingFrontAMI::distributePatches
} }
if (domain == Pstream::myProcNo(comm_)) if (domain == Pstream::myProcNo(comm()))
{ {
// Do send/receive for myself // Do send/receive for myself
faces[domain] = subPatch.localFaces(); faces[domain] = subPatch.localFaces();
points[domain] = subPatch.localPoints(); points[domain] = subPatch.localPoints();
faceIDs[domain] = faceIDs[domain] =
gi.toGlobal(Pstream::myProcNo(comm_), sendElems); gi.toGlobal(Pstream::myProcNo(comm()), sendElems);
} }
else else
{ {
@ -118,7 +118,7 @@ void Foam::advancingFrontAMI::distributePatches
str str
<< subPatch.localFaces() << subPatch.localFaces()
<< subPatch.localPoints() << subPatch.localPoints()
<< gi.toGlobal(Pstream::myProcNo(comm_), sendElems); << gi.toGlobal(Pstream::myProcNo(comm()), sendElems);
} }
} }
} }
@ -127,11 +127,11 @@ void Foam::advancingFrontAMI::distributePatches
// Consume // Consume
for (const int domain : Pstream::allProcs(comm_)) for (const int domain : Pstream::allProcs(comm()))
{ {
const labelList& recvElems = map.constructMap()[domain]; const labelList& recvElems = map.constructMap()[domain];
if (domain != Pstream::myProcNo(comm_) && recvElems.size()) if (domain != Pstream::myProcNo(comm()) && recvElems.size())
{ {
UIPstream is(domain, pBufs); UIPstream is(domain, pBufs);
@ -177,10 +177,10 @@ void Foam::advancingFrontAMI::distributeAndMergePatches
// My own data first // My own data first
{ {
const labelList& faceIDs = allTgtFaceIDs[Pstream::myProcNo(comm_)]; const labelList& faceIDs = allTgtFaceIDs[Pstream::myProcNo(comm())];
SubList<label>(tgtFaceIDs, faceIDs.size()) = faceIDs; SubList<label>(tgtFaceIDs, faceIDs.size()) = faceIDs;
const faceList& fcs = allFaces[Pstream::myProcNo(comm_)]; const faceList& fcs = allFaces[Pstream::myProcNo(comm())];
for (const face& f : fcs) for (const face& f : fcs)
{ {
face& newF = tgtFaces[nFaces++]; face& newF = tgtFaces[nFaces++];
@ -191,7 +191,7 @@ void Foam::advancingFrontAMI::distributeAndMergePatches
} }
} }
const pointField& pts = allPoints[Pstream::myProcNo(comm_)]; const pointField& pts = allPoints[Pstream::myProcNo(comm())];
for (const point& pt: pts) for (const point& pt: pts)
{ {
tgtPoints[nPoints++] = pt; tgtPoints[nPoints++] = pt;
@ -202,7 +202,7 @@ void Foam::advancingFrontAMI::distributeAndMergePatches
// Other proc data follows // Other proc data follows
forAll(allFaces, proci) forAll(allFaces, proci)
{ {
if (proci != Pstream::myProcNo(comm_)) if (proci != Pstream::myProcNo(comm()))
{ {
const labelList& faceIDs = allTgtFaceIDs[proci]; const labelList& faceIDs = allTgtFaceIDs[proci];
SubList<label>(tgtFaceIDs, faceIDs.size(), nFaces) = faceIDs; SubList<label>(tgtFaceIDs, faceIDs.size(), nFaces) = faceIDs;
@ -258,12 +258,19 @@ Foam::autoPtr<Foam::mapDistribute> Foam::advancingFrontAMI::calcProcMap
const primitivePatch& tgtPatch const primitivePatch& tgtPatch
) const ) const
{ {
if (comm() == -1)
{
FatalErrorInFunction
<< "Processor " << UPstream::myProcNo(UPstream::worldComm)
<< " not in communicator " << comm() << exit(FatalError);
}
// Get decomposition of patch // Get decomposition of patch
List<treeBoundBoxList> procBb(Pstream::nProcs(comm_)); List<treeBoundBoxList> procBb(Pstream::nProcs(comm()));
if (srcPatch.size()) if (srcPatch.size())
{ {
procBb[Pstream::myProcNo(comm_)] = procBb[Pstream::myProcNo(comm())] =
AABBTree<face> AABBTree<face>
( (
srcPatch.localFaces(), srcPatch.localFaces(),
@ -273,10 +280,10 @@ Foam::autoPtr<Foam::mapDistribute> Foam::advancingFrontAMI::calcProcMap
} }
else else
{ {
procBb[Pstream::myProcNo(comm_)] = treeBoundBoxList(); procBb[Pstream::myProcNo(comm())] = treeBoundBoxList();
} }
Pstream::allGatherList(procBb, UPstream::msgType(), comm_); Pstream::allGatherList(procBb, UPstream::msgType(), comm());
if (debug) if (debug)
{ {
@ -296,10 +303,10 @@ Foam::autoPtr<Foam::mapDistribute> Foam::advancingFrontAMI::calcProcMap
{ {
// Per processor indices into all segments to send // Per processor indices into all segments to send
List<DynamicList<label>> dynSendMap(Pstream::nProcs(comm_)); List<DynamicList<label>> dynSendMap(Pstream::nProcs(comm()));
// Work array - whether processor bb overlaps the face bounds // Work array - whether processor bb overlaps the face bounds
boolList procBbOverlaps(Pstream::nProcs(comm_)); boolList procBbOverlaps(Pstream::nProcs(comm()));
forAll(faces, facei) forAll(faces, facei)
{ {
@ -321,7 +328,7 @@ Foam::autoPtr<Foam::mapDistribute> Foam::advancingFrontAMI::calcProcMap
} }
// Convert dynamicList to labelList // Convert dynamicList to labelList
sendMap.setSize(Pstream::nProcs(comm_)); sendMap.setSize(Pstream::nProcs(comm()));
forAll(sendMap, proci) forAll(sendMap, proci)
{ {
sendMap[proci].transfer(dynSendMap[proci]); sendMap[proci].transfer(dynSendMap[proci]);
@ -344,7 +351,7 @@ Foam::autoPtr<Foam::mapDistribute> Foam::advancingFrontAMI::calcProcMap
std::move(sendMap), std::move(sendMap),
false, //subHasFlip false, //subHasFlip
false, //constructHasFlip false, //constructHasFlip
comm_ comm()
); );
} }

16
src/meshTools/AMIInterpolation/AMIInterpolation/faceAreaWeightAMI/faceAreaWeightAMI.C
View File

@ -711,9 +711,9 @@ bool Foam::faceAreaWeightAMI::calculate
tgtWeights_[i].transfer(tgtWght[i]); tgtWeights_[i].transfer(tgtWght[i]);
} }
if (distributed()) if (distributed() && comm() != -1)
{ {
const label myRank = UPstream::myProcNo(comm_); const label myRank = UPstream::myProcNo(comm());
// Allocate unique tag for all comms // Allocate unique tag for all comms
const int oldTag = UPstream::incrMsgType(); const int oldTag = UPstream::incrMsgType();
@ -721,8 +721,8 @@ bool Foam::faceAreaWeightAMI::calculate
const primitivePatch& tgtPatch0 = this->tgtPatch0(); const primitivePatch& tgtPatch0 = this->tgtPatch0();
// Create global indexing for each original patch // Create global indexing for each original patch
globalIndex globalSrcFaces(srcPatch0.size(), comm_); globalIndex globalSrcFaces(srcPatch0.size(), comm());
globalIndex globalTgtFaces(tgtPatch0.size(), comm_); globalIndex globalTgtFaces(tgtPatch0.size(), comm());
for (labelList& addressing : srcAddress_) for (labelList& addressing : srcAddress_)
{ {
@ -754,7 +754,7 @@ bool Foam::faceAreaWeightAMI::calculate
ListOps::appendEqOp<label>(), ListOps::appendEqOp<label>(),
flipOp(), // flip operation flipOp(), // flip operation
UPstream::msgType()+77431, UPstream::msgType()+77431,
comm_ comm()
); );
mapDistributeBase::distribute mapDistributeBase::distribute
@ -771,7 +771,7 @@ bool Foam::faceAreaWeightAMI::calculate
ListOps::appendEqOp<scalar>(), ListOps::appendEqOp<scalar>(),
flipOp(), flipOp(),
UPstream::msgType()+77432, UPstream::msgType()+77432,
comm_ comm()
); );
// Note: using patch face areas calculated by the AMI method // Note: using patch face areas calculated by the AMI method
@ -787,7 +787,7 @@ bool Foam::faceAreaWeightAMI::calculate
tgtAddress_, tgtAddress_,
cMapSrc, cMapSrc,
UPstream::msgType()+77433, UPstream::msgType()+77433,
comm_ comm()
) )
); );
@ -800,7 +800,7 @@ bool Foam::faceAreaWeightAMI::calculate
srcAddress_, srcAddress_,
cMapTgt, cMapTgt,
UPstream::msgType()+77434, UPstream::msgType()+77434,
comm_ comm()
) )
); );

16
src/meshTools/AMIInterpolation/AMIInterpolation/faceAreaWeightAMI2D/faceAreaWeightAMI2D.C
View File

@ -425,9 +425,9 @@ bool Foam::faceAreaWeightAMI2D::calculate
tgtWeights_[i].transfer(tgtWght[i]); tgtWeights_[i].transfer(tgtWght[i]);
} }
if (distributed()) if (distributed() && comm() != -1)
{ {
const label myRank = UPstream::myProcNo(comm_); const label myRank = UPstream::myProcNo(comm());
// Allocate unique tag for all comms // Allocate unique tag for all comms
const int oldTag = UPstream::incrMsgType(); const int oldTag = UPstream::incrMsgType();
@ -435,8 +435,8 @@ bool Foam::faceAreaWeightAMI2D::calculate
const primitivePatch& tgtPatch0 = this->tgtPatch0(); const primitivePatch& tgtPatch0 = this->tgtPatch0();
// Create global indexing for each original patch // Create global indexing for each original patch
const globalIndex globalSrcFaces(srcPatch0.size(), comm_); const globalIndex globalSrcFaces(srcPatch0.size(), comm());
const globalIndex globalTgtFaces(tgtPatch0.size(), comm_); const globalIndex globalTgtFaces(tgtPatch0.size(), comm());
for (labelList& addressing : srcAddress_) for (labelList& addressing : srcAddress_)
{ {
@ -468,7 +468,7 @@ bool Foam::faceAreaWeightAMI2D::calculate
ListOps::appendEqOp<label>(), ListOps::appendEqOp<label>(),
flipOp(), // flip operation flipOp(), // flip operation
UPstream::msgType()+77431, UPstream::msgType()+77431,
comm_ comm()
); );
mapDistributeBase::distribute mapDistributeBase::distribute
@ -485,7 +485,7 @@ bool Foam::faceAreaWeightAMI2D::calculate
ListOps::appendEqOp<scalar>(), ListOps::appendEqOp<scalar>(),
flipOp(), // flip operation flipOp(), // flip operation
UPstream::msgType()+77432, UPstream::msgType()+77432,
comm_ comm()
); );
// Note: using patch face areas calculated by the AMI method // Note: using patch face areas calculated by the AMI method
@ -501,7 +501,7 @@ bool Foam::faceAreaWeightAMI2D::calculate
tgtAddress_, tgtAddress_,
cMapSrc, cMapSrc,
UPstream::msgType()+77433, UPstream::msgType()+77433,
comm_ comm()
) )
); );
@ -514,7 +514,7 @@ bool Foam::faceAreaWeightAMI2D::calculate
srcAddress_, srcAddress_,
cMapTgt, cMapTgt,
UPstream::msgType()+77434, UPstream::msgType()+77434,
comm_ comm()
) )
); );

34
src/meshTools/AMIInterpolation/AMIInterpolation/nearestFaceAMI/nearestFaceAMI.C
View File

@ -5,7 +5,7 @@
\\ / A nd | www.openfoam.com \\ / A nd | www.openfoam.com
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2020-2022 OpenCFD Ltd. Copyright (C) 2020-2024 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
@ -47,15 +47,15 @@ Foam::autoPtr<Foam::mapDistribute> Foam::nearestFaceAMI::calcFaceMap
) const ) const
{ {
// Generate the list of processor bounding boxes for tgtPatch // Generate the list of processor bounding boxes for tgtPatch
List<boundBox> procBbs(Pstream::nProcs()); List<boundBox> procBbs(Pstream::nProcs(comm()));
procBbs[Pstream::myProcNo()] = procBbs[Pstream::myProcNo(comm())] =
boundBox(tgtPatch.points(), tgtPatch.meshPoints(), true); boundBox(tgtPatch.points(), tgtPatch.meshPoints(), false);
Pstream::allGatherList(procBbs); Pstream::allGatherList(procBbs, UPstream::msgType(), comm());
// Identify which of my local src faces intersect with each processor // Identify which of my local src faces intersect with each processor
// tgtPatch bb within the current match's search distance // tgtPatch bb within the current match's search distance
const pointField& srcCcs = srcPatch.faceCentres(); const pointField& srcCcs = srcPatch.faceCentres();
List<DynamicList<label>> dynSendMap(Pstream::nProcs()); List<DynamicList<label>> dynSendMap(Pstream::nProcs(comm()));
forAll(localInfo, srcFacei) forAll(localInfo, srcFacei)
{ {
@ -66,7 +66,7 @@ Foam::autoPtr<Foam::mapDistribute> Foam::nearestFaceAMI::calcFaceMap
forAll(procBbs, proci) forAll(procBbs, proci)
{ {
if (proci != Pstream::myProcNo()) if (proci != Pstream::myProcNo(comm()))
{ {
if (procBbs[proci].overlaps(srcCcs[srcFacei], r2)) if (procBbs[proci].overlaps(srcCcs[srcFacei], r2))
{ {
@ -77,7 +77,7 @@ Foam::autoPtr<Foam::mapDistribute> Foam::nearestFaceAMI::calcFaceMap
} }
// Convert dynamicList to labelList // Convert dynamicList to labelList
labelListList sendMap(Pstream::nProcs()); labelListList sendMap(Pstream::nProcs(comm()));
forAll(sendMap, proci) forAll(sendMap, proci)
{ {
sendMap[proci].transfer(dynSendMap[proci]); sendMap[proci].transfer(dynSendMap[proci]);
@ -89,7 +89,13 @@ Foam::autoPtr<Foam::mapDistribute> Foam::nearestFaceAMI::calcFaceMap
} }
} }
return autoPtr<mapDistribute>::New(std::move(sendMap)); return autoPtr<mapDistribute>::New
(
std::move(sendMap),
false,
false,
comm()
);
} }
@ -108,10 +114,10 @@ Foam::autoPtr<Foam::mapDistribute> Foam::nearestFaceAMI::calcDistributed
} }
// Create global indexing for tgtPatch // Create global indexing for tgtPatch
globalIndex globalTgtCells(tgt.size()); globalIndex globalTgtCells(tgt.size(), comm());
const label myRank = UPstream::myProcNo(comm_); const label myRank = UPstream::myProcNo(comm());
// First pass // First pass
@ -205,7 +211,7 @@ Foam::autoPtr<Foam::mapDistribute> Foam::nearestFaceAMI::calcDistributed
nearestEqOp(), nearestEqOp(),
identityOp(), // No flipping identityOp(), // No flipping
UPstream::msgType(), UPstream::msgType(),
comm_ comm()
); );
@ -234,7 +240,7 @@ Foam::autoPtr<Foam::mapDistribute> Foam::nearestFaceAMI::calcDistributed
srcToTgtAddr, srcToTgtAddr,
cMap, cMap,
UPstream::msgType(), UPstream::msgType(),
comm_ comm()
); );
} }
@ -297,7 +303,7 @@ bool Foam::nearestFaceAMI::calculate
// TODO: implement symmetric calculation controls; assume yes for now // TODO: implement symmetric calculation controls; assume yes for now
bool symmetric_ = true; bool symmetric_ = true;
if (this->distributed()) if (this->distributed() && comm() != -1)
{ {
tgtMapPtr_ = tgtMapPtr_ =
calcDistributed calcDistributed

4
src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicACMIGAMGInterfaceField/cyclicACMIGAMGInterfaceField.C
View File

@ -184,7 +184,7 @@ void Foam::cyclicACMIGAMGInterfaceField::initInterfaceMatrixUpdate
: cyclicACMIInterface_.neighbPatch().AMI() : cyclicACMIInterface_.neighbPatch().AMI()
); );
if (AMI.distributed()) if (AMI.distributed() && AMI.comm() != -1)
{ {
DebugPout<< "cyclicACMIFvPatchField::initInterfaceMatrixUpdate() :" DebugPout<< "cyclicACMIFvPatchField::initInterfaceMatrixUpdate() :"
<< " interface:" << cyclicACMIInterface_.index() << " interface:" << cyclicACMIInterface_.index()
@ -277,7 +277,7 @@ void Foam::cyclicACMIGAMGInterfaceField::updateInterfaceMatrix
<< endl; << endl;
if (AMI.distributed()) if (AMI.distributed() && AMI.comm() != -1)
{ {
const auto& map = const auto& map =
( (

4
src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicAMIGAMGInterfaceField/cyclicAMIGAMGInterfaceField.C
View File

@ -184,7 +184,7 @@ void Foam::cyclicAMIGAMGInterfaceField::initInterfaceMatrixUpdate
: cyclicAMIInterface_.neighbPatch().AMI() : cyclicAMIInterface_.neighbPatch().AMI()
); );
if (AMI.distributed()) if (AMI.distributed() && AMI.comm() != -1)
{ {
//DebugPout<< "cyclicAMIFvPatchField::initInterfaceMatrixUpdate() :" //DebugPout<< "cyclicAMIFvPatchField::initInterfaceMatrixUpdate() :"
// << " interface:" << cyclicAMIInterface_.index() // << " interface:" << cyclicAMIInterface_.index()
@ -284,7 +284,7 @@ void Foam::cyclicAMIGAMGInterfaceField::updateInterfaceMatrix
// << " AMI low-weight:" << AMI.applyLowWeightCorrection() // << " AMI low-weight:" << AMI.applyLowWeightCorrection()
// << endl; // << endl;
if (AMI.distributed()) if (AMI.distributed() && AMI.comm() != -1)
{ {
if (commsType != UPstream::commsTypes::nonBlocking) if (commsType != UPstream::commsTypes::nonBlocking)
{ {

315
src/meshTools/AMIInterpolation/GAMG/interfaces/cyclicACMIGAMGInterface/cyclicACMIGAMGInterface.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2013-2016 OpenFOAM Foundation Copyright (C) 2013-2016 OpenFOAM Foundation
Copyright (C) 2019,2023 OpenCFD Ltd. Copyright (C) 2019-2025 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
@ -84,7 +84,8 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
reverseT_ reverseT_
( (
refCast<const cyclicACMILduInterface>(fineInterface).reverseT() refCast<const cyclicACMILduInterface>(fineInterface).reverseT()
) ),
myProcNo_(-1)
{ {
const auto& fineCyclicACMIInterface = const auto& fineCyclicACMIInterface =
refCast<const cyclicACMILduInterface>(fineInterface); refCast<const cyclicACMILduInterface>(fineInterface);
@ -190,13 +191,24 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
neighbPatchID_(readLabel(is)), neighbPatchID_(readLabel(is)),
owner_(readBool(is)), owner_(readBool(is)),
forwardT_(is), forwardT_(is),
reverseT_(is) reverseT_(is),
myProcNo_(-1)
{ {
const bool hasAMI(readBool(is)); const bool hasAMI(readBool(is));
if (hasAMI) if (hasAMI)
{ {
amiPtr_.reset(new AMIPatchToPatchInterpolation(is)); amiPtr_.reset(new AMIPatchToPatchInterpolation(is));
// Store originating ranks locally - used when processor agglomerating
// onto a processor that wasn't in the communicator originally (since
// it had no faces)
const label comm = AMI().comm();
if (comm != -1)
{
is >> myProcNo_;
}
} }
} }
@ -244,27 +256,64 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
reverseT_ reverseT_
( (
refCast<const cyclicACMILduInterface>(fineInterface).reverseT() refCast<const cyclicACMILduInterface>(fineInterface).reverseT()
) ),
myProcNo_(-1)
{ {
const auto& fineCyclicACMIInterface = if (!owner_)
refCast<const cyclicACMIGAMGInterface>(fineInterface);
if (fineCyclicACMIInterface.amiPtr_)
{ {
const auto& AMI = const_cast<AMIPatchToPatchInterpolation&> return;
( }
fineCyclicACMIInterface.AMI()
);
label singlePatchProc = AMI.singlePatchProc();
// Get some sizes // Get stats, sizes from the input interfaces. For the global settings
label nSrc = 0; // the problem is that the
label nTgt = 0; // local processor might not have any valid interfaces so here just
// collect and do a global reduction afterwards.
// Structure to pack all. First element is used to decide who has the
// valid AMI.
typedef
Tuple2
<
label,
Tuple2
<
Tuple2
<
FixedList<bool, 4>,
scalar
>,
label
>
> AMIType;
AMIType globalInfo;
FixedList<bool, 4>& bools = globalInfo.second().first().first();
// Define aliases to make our life easier
label& firstValidAMI = globalInfo.first();
bool& requireMatch = bools[0];
bool& reverseTarget = bools[1];
bool& srcHasFlip = bools[2];
bool& tgtHasFlip = bools[3];
scalar& lowWeightCorrection = globalInfo.second().first().second();
label& singlePatchProc = globalInfo.second().second();
// Initialise all global variables
firstValidAMI = labelMax;
requireMatch = false;
reverseTarget = false;
srcHasFlip = false;
tgtHasFlip = false;
lowWeightCorrection = -1;
singlePatchProc = -1;
// Initialise all local variables
bool hasSrcMagSf = false; bool hasSrcMagSf = false;
bool hasSrcCentroids = false; bool hasSrcCentroids = false;
bool hasTgtMagSf = false; bool hasTgtMagSf = false;
label nSrc = 0;
label nTgt = 0;
forAll(allInterfaces, inti) forAll(allInterfaces, inti)
{ {
@ -272,7 +321,31 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
{ {
const auto& intf = const auto& intf =
refCast<const cyclicACMIGAMGInterface>(allInterfaces[inti]); refCast<const cyclicACMIGAMGInterface>(allInterfaces[inti]);
if (!intf.amiPtr_)
{
continue;
}
if (firstValidAMI == labelMax)
{
firstValidAMI = inti;
}
const auto& AMI = intf.AMI(); const auto& AMI = intf.AMI();
if (AMI.distributed() && AMI.comm() != -1)
{
singlePatchProc = -1;
srcHasFlip =
srcHasFlip || AMI.srcMap().constructHasFlip();
tgtHasFlip =
tgtHasFlip || AMI.tgtMap().constructHasFlip();
}
requireMatch = AMI.requireMatch();
reverseTarget = AMI.reverseTarget();
lowWeightCorrection = AMI.lowWeightCorrection();
nSrc += AMI.srcAddress().size(); nSrc += AMI.srcAddress().size();
nTgt += AMI.tgtAddress().size(); nTgt += AMI.tgtAddress().size();
@ -313,6 +386,39 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
} }
// Reduce global information in case one of the coarse ranks does not
// have an input AMI to get data from. Could use minFirstEqOp from Tuple2
// instead ...
Pstream::combineReduce
(
globalInfo,
[](AMIType& x, const AMIType& y)
{
if (y.first() < x.first())
{
x = y;
}
},
Pstream::msgType(),
coarseComm
);
DebugPout
<< "Input amis :"
<< " singlePatchProc:" << singlePatchProc
<< " srcHasFlip:" << srcHasFlip
<< " tgtHasFlip:" << tgtHasFlip
<< " requireMatch:" << requireMatch
<< " reverseTarget:" << reverseTarget
<< " lowWeightCorrection:" << lowWeightCorrection
<< " hasSrcMagSf:" << hasSrcMagSf
<< " hasSrcCentroids:" << hasSrcCentroids
<< " hasTgtMagSf:" << hasTgtMagSf
<< " nSrc:" << nSrc
<< " nTgt:" << nTgt
<< endl;
labelListList srcAddress; labelListList srcAddress;
scalarListList srcWeights; scalarListList srcWeights;
scalarList srcMagSf; scalarList srcMagSf;
@ -330,8 +436,81 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
// Map to send tgt side data to src side // Map to send tgt side data to src side
autoPtr<mapDistribute> tgtToSrcMap; autoPtr<mapDistribute> tgtToSrcMap;
if (AMI.distributed()) if (singlePatchProc == -1)
{ {
// Find ranks that agglomerate together
const label myAgglom = UPstream::myProcNo(coarseComm);
// Per input map either -1 or the index in the maps that is local
// data.
labelList localRanks(allInterfaces.size(), -1);
// From rank in coarse communicator back to rank in original (fine)
// communicator.
labelListList newToOldRanks;
{
// Pass 1: count number of valid maps
label nOldRanks = 0;
forAll(allInterfaces, inti)
{
if (allInterfaces.set(inti))
{
const auto& intf = refCast<const cyclicACMIGAMGInterface>
(
allInterfaces[inti]
);
if (!intf.amiPtr_ || intf.AMI().comm() == -1)
{
continue;
}
nOldRanks++;
}
}
// Pass 2: collect
DynamicList<label> oldRanks(nOldRanks);
forAll(allInterfaces, inti)
{
if (allInterfaces.set(inti))
{
const auto& intf = refCast<const cyclicACMIGAMGInterface>
(
allInterfaces[inti]
);
if (!intf.amiPtr_ || intf.AMI().comm() == -1)
{
continue;
}
label fineRank = -1;
if (intf.myProcNo() == -1)
{
// The interface was already local so got never
// sent across so myProcNo_ is never set ...
fineRank = UPstream::myProcNo(intf.AMI().comm());
}
else
{
fineRank = intf.myProcNo();
}
oldRanks.append(fineRank);
localRanks[inti] = fineRank;
}
}
// Pull individual parts together - this is the only communication
// needed.
newToOldRanks = Pstream::listGatherValues
(
labelList(std::move(oldRanks)),
coarseComm
);
Pstream::broadcast(newToOldRanks, coarseComm);
}
// Create combined maps // Create combined maps
UPtrList<const mapDistribute> srcMaps(allInterfaces.size()); UPtrList<const mapDistribute> srcMaps(allInterfaces.size());
UPtrList<const mapDistribute> tgtMaps(allInterfaces.size()); UPtrList<const mapDistribute> tgtMaps(allInterfaces.size());
@ -343,27 +522,22 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
( (
allInterfaces[inti] allInterfaces[inti]
); );
if (!intf.amiPtr_)
{
// Should not be in allInterfaces?
continue;
}
const auto& AMI = intf.AMI(); const auto& AMI = intf.AMI();
if (AMI.comm() != -1)
{
srcMaps.set(inti, &AMI.srcMap()); srcMaps.set(inti, &AMI.srcMap());
tgtMaps.set(inti, &AMI.tgtMap()); tgtMaps.set(inti, &AMI.tgtMap());
} }
} }
}
// Find ranks that agglomerate together
const label myAgglom =
procAgglomMap[UPstream::myProcNo(AMI.comm())];
// Invert procAgglomMap
const labelListList newToOldRanks
(
invertOneToMany
(
UPstream::nProcs(coarseComm),
procAgglomMap
)
);
const labelList& localRanks = newToOldRanks[myAgglom];
// Offsets for slots into results of srcToTgtMap // Offsets for slots into results of srcToTgtMap
@ -375,7 +549,7 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
new mapDistribute new mapDistribute
( (
srcMaps, srcMaps,
localRanks, // per src map which rank it is from localRanks, // per src map which rank represents local data
coarseComm, coarseComm,
newToOldRanks, // destination rank to source ranks newToOldRanks, // destination rank to source ranks
srcStartOfLocal, srcStartOfLocal,
@ -383,6 +557,7 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
) )
); );
// Assemble tgtAddress // Assemble tgtAddress
tgtAddress.setSize(nTgt); tgtAddress.setSize(nTgt);
if (tgtAddress.size()) if (tgtAddress.size())
@ -392,11 +567,16 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
{ {
if (allInterfaces.set(inti)) if (allInterfaces.set(inti))
{ {
const auto& intf = const auto& intf = refCast<const cyclicACMIGAMGInterface>
refCast<const cyclicACMIGAMGInterface>
( (
allInterfaces[inti] allInterfaces[inti]
); );
if (!intf.amiPtr_)
{
continue;
}
const auto& AMI = intf.AMI(); const auto& AMI = intf.AMI();
const auto& tgtSlots = AMI.tgtAddress(); const auto& tgtSlots = AMI.tgtAddress();
const label localSize = const label localSize =
@ -416,7 +596,7 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
localSize, localSize,
srcStartOfLocal[inti], srcStartOfLocal[inti],
srcCompactMaps[inti], srcCompactMaps[inti],
AMI.srcMap().constructHasFlip() //hasFlip srcHasFlip //hasFlip
); );
for (const label slot : newSlots) for (const label slot : newSlots)
@ -459,6 +639,7 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
) )
); );
// Assemble srcAddress // Assemble srcAddress
srcAddress.setSize(nSrc); srcAddress.setSize(nSrc);
if (srcAddress.size()) if (srcAddress.size())
@ -468,11 +649,16 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
{ {
if (allInterfaces.set(inti)) if (allInterfaces.set(inti))
{ {
const auto& intf = const auto& intf = refCast<const cyclicACMIGAMGInterface>
refCast<const cyclicACMIGAMGInterface>
( (
allInterfaces[inti] allInterfaces[inti]
); );
if (!intf.amiPtr_)
{
continue;
}
const auto& AMI = intf.AMI(); const auto& AMI = intf.AMI();
const auto& srcSlots = AMI.srcAddress(); const auto& srcSlots = AMI.srcAddress();
const label localSize = const label localSize =
@ -491,7 +677,7 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
localSize, localSize,
tgtStartOfLocal[inti], tgtStartOfLocal[inti],
tgtCompactMaps[inti], tgtCompactMaps[inti],
AMI.tgtMap().constructHasFlip() //hasFlip tgtHasFlip
); );
for (const label slot : newSlots) for (const label slot : newSlots)
@ -543,10 +729,14 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
} }
} }
if (!usesRemote) // We can't have a single rank become fully-local since we
// expect singlePatchProc to be synchronised. So make sure all
// have become local
if (!returnReduceOr(usesRemote, coarseComm))
{ {
//Pout<< "** making fully local on new rank " DebugPout<< "** making fully local on new rank "
// << myAgglom << " in comm:" << coarseComm << endl; << myAgglom << " in comm:" << coarseComm << endl;
singlePatchProc = myAgglom; singlePatchProc = myAgglom;
srcToTgtMap.clear(); srcToTgtMap.clear();
tgtToSrcMap.clear(); tgtToSrcMap.clear();
@ -570,6 +760,12 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
( (
allInterfaces[inti] allInterfaces[inti]
); );
if (!intf.amiPtr_)
{
continue;
}
const auto& AMI = intf.AMI(); const auto& AMI = intf.AMI();
const auto& srcA = AMI.srcAddress(); const auto& srcA = AMI.srcAddress();
@ -625,6 +821,12 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
( (
allInterfaces[inti] allInterfaces[inti]
); );
if (!intf.amiPtr_)
{
continue;
}
const auto& AMI = intf.AMI(); const auto& AMI = intf.AMI();
const auto& srcA = AMI.srcAddress(); const auto& srcA = AMI.srcAddress();
@ -672,9 +874,9 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
( (
new AMIPatchToPatchInterpolation new AMIPatchToPatchInterpolation
( (
AMI.requireMatch(), requireMatch,
AMI.reverseTarget(), reverseTarget,
AMI.lowWeightCorrection() lowWeightCorrection
) )
); );
amiPtr_().comm(coarseComm), amiPtr_().comm(coarseComm),
@ -691,14 +893,18 @@ Foam::cyclicACMIGAMGInterface::cyclicACMIGAMGInterface
amiPtr_().srcMagSf() = std::move(srcMagSf); amiPtr_().srcMagSf() = std::move(srcMagSf);
amiPtr_().srcCentroids() = std::move(srcCentroids); amiPtr_().srcCentroids() = std::move(srcCentroids);
amiPtr_().tgtMagSf() = std::move(tgtMagSf); amiPtr_().tgtMagSf() = std::move(tgtMagSf);
}
//Pout<< "** constructed new ami:"
// << " comm:" << amiPtr_().comm()
// << " srcMap.comm:" << amiPtr_().srcMap().comm()
// << " tgtMap.comm:" << amiPtr_().tgtMap().comm()
// << endl;
} }
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // // * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
Foam::tmp<Foam::labelField> Foam::tmp<Foam::labelField> Foam::cyclicACMIGAMGInterface::internalFieldTransfer
Foam::cyclicACMIGAMGInterface::internalFieldTransfer
( (
const Pstream::commsTypes commsType, const Pstream::commsTypes commsType,
const labelUList& iF const labelUList& iF
@ -736,6 +942,15 @@ void Foam::cyclicACMIGAMGInterface::write(Ostream& os) const
{ {
os << token::SPACE; os << token::SPACE;
AMI().writeData(os); AMI().writeData(os);
// Write processors in communicator
const label comm = AMI().comm();
if (comm != -1)
{
os << token::SPACE
<< UPstream::myProcNo(comm);
}
} }
} }

15
src/meshTools/AMIInterpolation/GAMG/interfaces/cyclicACMIGAMGInterface/cyclicACMIGAMGInterface.H
View File

@ -35,8 +35,8 @@ SourceFiles
\*---------------------------------------------------------------------------*/ \*---------------------------------------------------------------------------*/
#ifndef cyclicACMIGAMGInterface_H #ifndef Foam_cyclicACMIGAMGInterface_H
#define cyclicACMIGAMGInterface_H #define Foam_cyclicACMIGAMGInterface_H
#include "GAMGInterface.H" #include "GAMGInterface.H"
#include "cyclicACMILduInterface.H" #include "cyclicACMILduInterface.H"
@ -68,6 +68,11 @@ class cyclicACMIGAMGInterface
//- AMI interface //- AMI interface
autoPtr<AMIPatchToPatchInterpolation> amiPtr_; autoPtr<AMIPatchToPatchInterpolation> amiPtr_;
//- rank in AMI with respect to parent communicator.
//- Used for combining contributions from different processors if the
//- destination processor was not in the original set.
label myProcNo_;
// Private Member Functions // Private Member Functions
@ -213,6 +218,12 @@ public:
return reverseT_; return reverseT_;
} }
//- -1 or old local rank
virtual label myProcNo() const
{
return myProcNo_;
}
// I/O // I/O

317
src/meshTools/AMIInterpolation/GAMG/interfaces/cyclicAMIGAMGInterface/cyclicAMIGAMGInterface.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2011-2016 OpenFOAM Foundation Copyright (C) 2011-2016 OpenFOAM Foundation
Copyright (C) 2019,2023 OpenCFD Ltd. Copyright (C) 2019-2025 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
@ -84,7 +84,8 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
reverseT_ reverseT_
( (
refCast<const cyclicAMILduInterface>(fineInterface).reverseT() refCast<const cyclicAMILduInterface>(fineInterface).reverseT()
) ),
myProcNo_(-1)
{ {
const auto& fineCyclicAMIInterface = const auto& fineCyclicAMIInterface =
refCast<const cyclicAMILduInterface>(fineInterface); refCast<const cyclicAMILduInterface>(fineInterface);
@ -178,7 +179,8 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
const auto& AMI = amiPtr_(); const auto& AMI = amiPtr_();
if (debug & 2)
if (debug & 2 && AMI.comm() != -1)
{ {
const auto oldWarnComm = UPstream::commWarn(AMI.comm()); const auto oldWarnComm = UPstream::commWarn(AMI.comm());
@ -204,7 +206,7 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
const auto& weights = AMI.srcWeights(); const auto& weights = AMI.srcWeights();
labelList globalIDs(identity(nbrSize)); labelList globalIDs(identity(nbrSize));
if (AMI.distributed()) if (AMI.distributed() && AMI.comm() != -1)
{ {
const auto& map = AMI.tgtMap(); const auto& map = AMI.tgtMap();
forAll(map.subMap(), proci) forAll(map.subMap(), proci)
@ -254,7 +256,7 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
const auto& weights = AMI.tgtWeights(); const auto& weights = AMI.tgtWeights();
labelList globalIDs(identity(this->size())); labelList globalIDs(identity(this->size()));
if (AMI.distributed()) if (AMI.distributed() && AMI.comm() != -1)
{ {
const auto& map = AMI.srcMap(); const auto& map = AMI.srcMap();
forAll(map.subMap(), proci) forAll(map.subMap(), proci)
@ -313,13 +315,24 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
neighbPatchID_(readLabel(is)), neighbPatchID_(readLabel(is)),
owner_(readBool(is)), owner_(readBool(is)),
forwardT_(is), forwardT_(is),
reverseT_(is) reverseT_(is),
myProcNo_(-1)
{ {
const bool hasAMI(readBool(is)); const bool hasAMI(readBool(is));
if (hasAMI) if (hasAMI)
{ {
amiPtr_.reset(new AMIPatchToPatchInterpolation(is)); amiPtr_.reset(new AMIPatchToPatchInterpolation(is));
// Store originating ranks locally - used when processor agglomerating
// onto a processor that wasn't in the communicator originally (since
// it had no faces)
const label comm = AMI().comm();
if (comm != -1)
{
is >> myProcNo_;
}
} }
} }
@ -367,28 +380,64 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
reverseT_ reverseT_
( (
refCast<const cyclicAMILduInterface>(fineInterface).reverseT() refCast<const cyclicAMILduInterface>(fineInterface).reverseT()
) ),
myProcNo_(-1)
{ {
const auto& fineCyclicAMIInterface = if (!owner_)
refCast<const cyclicAMIGAMGInterface>(fineInterface);
if (fineCyclicAMIInterface.amiPtr_)
{ {
const auto& fineAMI = const_cast<AMIPatchToPatchInterpolation&> return;
( }
fineCyclicAMIInterface.AMI()
);
label singlePatchProc = fineAMI.singlePatchProc();
// Get stats, sizes from the input interfaces. For the global settings
// the problem is that the
// local processor might not have any valid interfaces so here just
// collect and do a global reduction afterwards.
// Get some sizes // Structure to pack all. First element is used to decide who has the
label nSrc = 0; // valid AMI.
label nTgt = 0; typedef
Tuple2
<
label,
Tuple2
<
Tuple2
<
FixedList<bool, 4>,
scalar
>,
label
>
> AMIType;
AMIType globalInfo;
FixedList<bool, 4>& bools = globalInfo.second().first().first();
// Define aliases to make our life easier
label& firstValidAMI = globalInfo.first();
bool& requireMatch = bools[0];
bool& reverseTarget = bools[1];
bool& srcHasFlip = bools[2];
bool& tgtHasFlip = bools[3];
scalar& lowWeightCorrection = globalInfo.second().first().second();
label& singlePatchProc = globalInfo.second().second();
// Initialise all global variables
firstValidAMI = labelMax;
requireMatch = false;
reverseTarget = false;
srcHasFlip = false;
tgtHasFlip = false;
lowWeightCorrection = -1;
singlePatchProc = -1;
// Initialise all local variables
bool hasSrcMagSf = false; bool hasSrcMagSf = false;
bool hasSrcCentroids = false; bool hasSrcCentroids = false;
bool hasTgtMagSf = false; bool hasTgtMagSf = false;
label nSrc = 0;
label nTgt = 0;
forAll(allInterfaces, inti) forAll(allInterfaces, inti)
{ {
@ -396,7 +445,31 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
{ {
const auto& intf = const auto& intf =
refCast<const cyclicAMIGAMGInterface>(allInterfaces[inti]); refCast<const cyclicAMIGAMGInterface>(allInterfaces[inti]);
if (!intf.amiPtr_)
{
continue;
}
if (firstValidAMI == labelMax)
{
firstValidAMI = inti;
}
const auto& AMI = intf.AMI(); const auto& AMI = intf.AMI();
if (AMI.distributed() && AMI.comm() != -1)
{
singlePatchProc = -1;
srcHasFlip =
srcHasFlip || AMI.srcMap().constructHasFlip();
tgtHasFlip =
tgtHasFlip || AMI.tgtMap().constructHasFlip();
}
requireMatch = AMI.requireMatch();
reverseTarget = AMI.reverseTarget();
lowWeightCorrection = AMI.lowWeightCorrection();
nSrc += AMI.srcAddress().size(); nSrc += AMI.srcAddress().size();
nTgt += AMI.tgtAddress().size(); nTgt += AMI.tgtAddress().size();
@ -437,6 +510,39 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
} }
// Reduce global information in case one of the coarse ranks does not
// have an input AMI to get data from. Could use minFirstEqOp from Tuple2
// instead ...
Pstream::combineReduce
(
globalInfo,
[](AMIType& x, const AMIType& y)
{
if (y.first() < x.first())
{
x = y;
}
},
Pstream::msgType(),
coarseComm
);
DebugPout
<< "Input amis :"
<< " singlePatchProc:" << singlePatchProc
<< " srcHasFlip:" << srcHasFlip
<< " tgtHasFlip:" << tgtHasFlip
<< " requireMatch:" << requireMatch
<< " reverseTarget:" << reverseTarget
<< " lowWeightCorrection:" << lowWeightCorrection
<< " hasSrcMagSf:" << hasSrcMagSf
<< " hasSrcCentroids:" << hasSrcCentroids
<< " hasTgtMagSf:" << hasTgtMagSf
<< " nSrc:" << nSrc
<< " nTgt:" << nTgt
<< endl;
labelListList srcAddress; labelListList srcAddress;
scalarListList srcWeights; scalarListList srcWeights;
scalarList srcMagSf; scalarList srcMagSf;
@ -454,8 +560,81 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
// Map to send tgt side data to src side // Map to send tgt side data to src side
autoPtr<mapDistribute> tgtToSrcMap; autoPtr<mapDistribute> tgtToSrcMap;
if (fineAMI.distributed()) if (singlePatchProc == -1)
{ {
// Find ranks that agglomerate together
const label myAgglom = UPstream::myProcNo(coarseComm);
// Per input map either -1 or the index in the maps that is local
// data.
labelList localRanks(allInterfaces.size(), -1);
// From rank in coarse communicator back to rank in original (fine)
// communicator.
labelListList newToOldRanks;
{
// Pass 1: count number of valid maps
label nOldRanks = 0;
forAll(allInterfaces, inti)
{
if (allInterfaces.set(inti))
{
const auto& intf = refCast<const cyclicAMIGAMGInterface>
(
allInterfaces[inti]
);
if (!intf.amiPtr_ || intf.AMI().comm() == -1)
{
continue;
}
nOldRanks++;
}
}
// Pass 2: collect
DynamicList<label> oldRanks(nOldRanks);
forAll(allInterfaces, inti)
{
if (allInterfaces.set(inti))
{
const auto& intf = refCast<const cyclicAMIGAMGInterface>
(
allInterfaces[inti]
);
if (!intf.amiPtr_ || intf.AMI().comm() == -1)
{
continue;
}
label fineRank = -1;
if (intf.myProcNo() == -1)
{
// The interface was already local so got never
// sent across so myProcNo_ is never set ...
fineRank = UPstream::myProcNo(intf.AMI().comm());
}
else
{
fineRank = intf.myProcNo();
}
oldRanks.append(fineRank);
localRanks[inti] = fineRank;
}
}
// Pull individual parts together - this is the only communication
// needed.
newToOldRanks = Pstream::listGatherValues
(
labelList(std::move(oldRanks)),
coarseComm
);
Pstream::broadcast(newToOldRanks, coarseComm);
}
// Create combined maps // Create combined maps
UPtrList<const mapDistribute> srcMaps(allInterfaces.size()); UPtrList<const mapDistribute> srcMaps(allInterfaces.size());
UPtrList<const mapDistribute> tgtMaps(allInterfaces.size()); UPtrList<const mapDistribute> tgtMaps(allInterfaces.size());
@ -467,27 +646,22 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
( (
allInterfaces[inti] allInterfaces[inti]
); );
if (!intf.amiPtr_)
{
// Should not be in allInterfaces?
continue;
}
const auto& AMI = intf.AMI(); const auto& AMI = intf.AMI();
if (AMI.comm() != -1)
{
srcMaps.set(inti, &AMI.srcMap()); srcMaps.set(inti, &AMI.srcMap());
tgtMaps.set(inti, &AMI.tgtMap()); tgtMaps.set(inti, &AMI.tgtMap());
} }
} }
}
// Find ranks that agglomerate together
const label myAgglom =
procAgglomMap[UPstream::myProcNo(fineAMI.comm())];
// Invert procAgglomMap
const labelListList newToOldRanks
(
invertOneToMany
(
UPstream::nProcs(coarseComm),
procAgglomMap
)
);
const labelList& localRanks = newToOldRanks[myAgglom];
// Offsets for slots into results of srcToTgtMap // Offsets for slots into results of srcToTgtMap
@ -499,7 +673,7 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
new mapDistribute new mapDistribute
( (
srcMaps, srcMaps,
localRanks, // per src map which rank it is from localRanks, // per src map which rank represents local data
coarseComm, coarseComm,
newToOldRanks, // destination rank to source ranks newToOldRanks, // destination rank to source ranks
srcStartOfLocal, srcStartOfLocal,
@ -507,6 +681,7 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
) )
); );
// Assemble tgtAddress // Assemble tgtAddress
tgtAddress.setSize(nTgt); tgtAddress.setSize(nTgt);
if (tgtAddress.size()) if (tgtAddress.size())
@ -520,6 +695,12 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
( (
allInterfaces[inti] allInterfaces[inti]
); );
if (!intf.amiPtr_)
{
continue;
}
const auto& AMI = intf.AMI(); const auto& AMI = intf.AMI();
const auto& tgtSlots = AMI.tgtAddress(); const auto& tgtSlots = AMI.tgtAddress();
const label localSize = const label localSize =
@ -539,7 +720,7 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
localSize, localSize,
srcStartOfLocal[inti], srcStartOfLocal[inti],
srcCompactMaps[inti], srcCompactMaps[inti],
AMI.srcMap().constructHasFlip() //hasFlip srcHasFlip //hasFlip
); );
for (const label slot : newSlots) for (const label slot : newSlots)
@ -582,6 +763,7 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
) )
); );
// Assemble srcAddress // Assemble srcAddress
srcAddress.setSize(nSrc); srcAddress.setSize(nSrc);
if (srcAddress.size()) if (srcAddress.size())
@ -595,6 +777,12 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
( (
allInterfaces[inti] allInterfaces[inti]
); );
if (!intf.amiPtr_)
{
continue;
}
const auto& AMI = intf.AMI(); const auto& AMI = intf.AMI();
const auto& srcSlots = AMI.srcAddress(); const auto& srcSlots = AMI.srcAddress();
const label localSize = const label localSize =
@ -613,7 +801,7 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
localSize, localSize,
tgtStartOfLocal[inti], tgtStartOfLocal[inti],
tgtCompactMaps[inti], tgtCompactMaps[inti],
AMI.tgtMap().constructHasFlip() //hasFlip tgtHasFlip
); );
for (const label slot : newSlots) for (const label slot : newSlots)
@ -665,10 +853,14 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
} }
} }
if (!usesRemote) // We can't have a single rank become fully-local since we
// expect singlePatchProc to be synchronised. So make sure all
// have become local
if (!returnReduceOr(usesRemote, coarseComm))
{ {
//Pout<< "** making fully local on new rank " DebugPout<< "** making fully local on new rank "
// << myAgglom << " in comm:" << coarseComm << endl; << myAgglom << " in comm:" << coarseComm << endl;
singlePatchProc = myAgglom; singlePatchProc = myAgglom;
srcToTgtMap.clear(); srcToTgtMap.clear();
tgtToSrcMap.clear(); tgtToSrcMap.clear();
@ -692,6 +884,12 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
( (
allInterfaces[inti] allInterfaces[inti]
); );
if (!intf.amiPtr_)
{
continue;
}
const auto& AMI = intf.AMI(); const auto& AMI = intf.AMI();
const auto& srcA = AMI.srcAddress(); const auto& srcA = AMI.srcAddress();
@ -747,6 +945,13 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
( (
allInterfaces[inti] allInterfaces[inti]
); );
if (!intf.amiPtr_)
{
continue;
}
const auto oldWarnComm = UPstream::commWarn(AMI.comm());
const auto& AMI = intf.AMI(); const auto& AMI = intf.AMI();
const auto& srcA = AMI.srcAddress(); const auto& srcA = AMI.srcAddress();
@ -794,9 +999,9 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
( (
new AMIPatchToPatchInterpolation new AMIPatchToPatchInterpolation
( (
fineAMI.requireMatch(), requireMatch,
fineAMI.reverseTarget(), reverseTarget,
fineAMI.lowWeightCorrection() lowWeightCorrection
) )
); );
amiPtr_().comm(coarseComm), amiPtr_().comm(coarseComm),
@ -819,7 +1024,8 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
{ {
const auto& AMI = amiPtr_(); const auto& AMI = amiPtr_();
const auto oldWarnComm = UPstream::commWarn(AMI.comm()); const auto oldWarnComm = UPstream::warnComm;
UPstream::warnComm = AMI.comm();
const label myRank = UPstream::myProcNo(AMI.comm()); const label myRank = UPstream::myProcNo(AMI.comm());
Pout<< "PROCAGGLOMERATED :" Pout<< "PROCAGGLOMERATED :"
@ -838,7 +1044,7 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
const auto& weights = AMI.srcWeights(); const auto& weights = AMI.srcWeights();
labelList globalIDs(identity(nbrSize)); labelList globalIDs(identity(nbrSize));
if (AMI.distributed()) if (AMI.distributed() && AMI.comm() != -1)
{ {
const auto& map = AMI.tgtMap(); const auto& map = AMI.tgtMap();
forAll(map.subMap(), proci) forAll(map.subMap(), proci)
@ -877,6 +1083,7 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
<< UIndirectList<label>(globalIDs, addresses[facei]) << UIndirectList<label>(globalIDs, addresses[facei])
<< endl; << endl;
} }
UPstream::commWarn(oldWarnComm);
} }
// From from owner to nbr side // From from owner to nbr side
{ {
@ -887,7 +1094,7 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
const auto& weights = AMI.tgtWeights(); const auto& weights = AMI.tgtWeights();
labelList globalIDs(identity(this->size())); labelList globalIDs(identity(this->size()));
if (AMI.distributed()) if (AMI.distributed() && AMI.comm() != -1)
{ {
const auto& map = AMI.srcMap(); const auto& map = AMI.srcMap();
forAll(map.subMap(), proci) forAll(map.subMap(), proci)
@ -928,8 +1135,7 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
} }
} }
Pout<< "DONE PROCAGGLOMERATED" << endl; Pout<< "DONE PROCAGGLOMERATED" << endl;
UPstream::commWarn(oldWarnComm); UPstream::warnComm = oldWarnComm;
}
} }
} }
@ -974,6 +1180,15 @@ void Foam::cyclicAMIGAMGInterface::write(Ostream& os) const
{ {
os << token::SPACE; os << token::SPACE;
AMI().writeData(os); AMI().writeData(os);
// Write processors in communicator
const label comm = AMI().comm();
if (comm != -1)
{
os << token::SPACE
<< UPstream::myProcNo(comm);
}
} }
} }

11
src/meshTools/AMIInterpolation/GAMG/interfaces/cyclicAMIGAMGInterface/cyclicAMIGAMGInterface.H
View File

@ -68,6 +68,11 @@ class cyclicAMIGAMGInterface
//- AMI interface //- AMI interface
autoPtr<AMIPatchToPatchInterpolation> amiPtr_; autoPtr<AMIPatchToPatchInterpolation> amiPtr_;
//- rank in AMI with respect to parent communicator.
//- Used for combining contributions from different processors if the
//- destination processor was not in the original set.
label myProcNo_;
// Private Member Functions // Private Member Functions
@ -213,6 +218,12 @@ public:
return reverseT_; return reverseT_;
} }
//- -1 or old local rank
virtual label myProcNo() const
{
return myProcNo_;
}
// I/O // I/O

16
src/meshTools/AMIInterpolation/patches/cyclicAMI/cyclicAMIPolyPatch/cyclicAMIPolyPatchTemplates.C
View File

@ -183,7 +183,7 @@ void Foam::cyclicAMIPolyPatch::initInterpolateUntransformed
{ {
const auto& AMI = (owner() ? this->AMI() : neighbPatch().AMI()); const auto& AMI = (owner() ? this->AMI() : neighbPatch().AMI());
if (AMI.distributed()) if (AMI.distributed() && AMI.comm() != -1)
{ {
const auto& map = (owner() ? AMI.tgtMap() : AMI.srcMap()); const auto& map = (owner() ? AMI.tgtMap() : AMI.srcMap());
@ -213,7 +213,7 @@ void Foam::cyclicAMIPolyPatch::initInterpolate
{ {
const auto& AMI = (owner() ? this->AMI() : neighbPatch().AMI()); const auto& AMI = (owner() ? this->AMI() : neighbPatch().AMI());
if (!AMI.distributed()) if (!AMI.distributed() || AMI.comm() == -1)
{ {
return; return;
} }
@ -273,12 +273,20 @@ Foam::tmp<Foam::Field<Type>> Foam::cyclicAMIPolyPatch::interpolate
const UList<Type>& defaultValues const UList<Type>& defaultValues
) const ) const
{ {
auto tresult = tmp<Field<Type>>::New(this->size(), Zero);
const auto& AMI = (owner() ? this->AMI() : neighbPatch().AMI()); const auto& AMI = (owner() ? this->AMI() : neighbPatch().AMI());
const auto& map = (owner() ? AMI.tgtMap() : AMI.srcMap());
Field<Type> work; Field<Type> work;
if (AMI.distributed()) if (AMI.distributed())
{ {
if (AMI.comm() == -1)
{
return tresult;
}
const auto& map = (owner() ? AMI.tgtMap() : AMI.srcMap());
// Receive (= copy) data from buffers into work. TBD: receive directly // Receive (= copy) data from buffers into work. TBD: receive directly
// into slices of work. // into slices of work.
map.receive map.receive
@ -291,8 +299,6 @@ Foam::tmp<Foam::Field<Type>> Foam::cyclicAMIPolyPatch::interpolate
} }
const Field<Type>& fld = (AMI.distributed() ? work : localFld); const Field<Type>& fld = (AMI.distributed() ? work : localFld);
auto tresult = tmp<Field<Type>>::New(this->size(), Zero);
// Rotate fields (vector and non-spherical tensors) // Rotate fields (vector and non-spherical tensors)
constexpr bool transform_supported = is_rotational_vectorspace_v<Type>; constexpr bool transform_supported = is_rotational_vectorspace_v<Type>;

2
src/meshTools/AMIInterpolation/patches/cyclicAMI/cyclicAMIPolyPatch/cyclicAMIPolyPatchTopologyChange.C
View File

@ -312,7 +312,7 @@ void Foam::cyclicAMIPolyPatch::setAMIFaces()
autoPtr<mapDistribute> srcToTgtMap1; autoPtr<mapDistribute> srcToTgtMap1;
autoPtr<mapDistribute> tgtToSrcMap1; autoPtr<mapDistribute> tgtToSrcMap1;
if (AMIPtr_->distributed()) if (AMIPtr_->distributed() && AMIPtr_().comm() != -1)
{ {
// Parallel running // Parallel running

4
src/sampling/meshToMesh/meshToMeshTemplates.C
View File

@ -537,7 +537,7 @@ void Foam::meshToMesh::mapSrcToTgt
( (
AMIList[i].singlePatchProc(), AMIList[i].singlePatchProc(),
( (
AMIList[i].distributed() (AMIList[i].distributed() && AMIList[i].comm() != -1)
? AMIList[i].hasSrcMap() // pointer to map ? AMIList[i].hasSrcMap() // pointer to map
: nullptr : nullptr
), ),
@ -769,7 +769,7 @@ void Foam::meshToMesh::mapTgtToSrc
( (
AMIList[i].singlePatchProc(), AMIList[i].singlePatchProc(),
( (
AMIList[i].distributed() (AMIList[i].distributed() && AMIList[i].comm() != -1)
? AMIList[i].hasTgtMap() // pointer to map ? AMIList[i].hasTgtMap() // pointer to map
: nullptr : nullptr
), ),