zhyang-dev/openfoam
1
0
Fork 0
mirror of https://develop.openfoam.com/Development/openfoam.git synced 2025-11-28 03:28:01 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

BUG: multi-world: avoid on-the-fly comms. Fixes #2529

Browse Source 
Probably can get away with less tetBasePtIs() triggering
This commit is contained in:
mattijs
2022-07-20 13:34:58 +01:00
parent 66ddf0a104
commit 873f7aac2d
2 changed files with 64 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/meshTools/mappedPatches/mappedPolyPatch/mappedPatchBase.C
View File

@ -733,6 +733,12 @@ void Foam::mappedPatchBase::calcMapping() const
DebugInFunction << nl;
// Early construction of tetBasePtIs since does parallel
// communication which might conflict with inter-world communicator
// (for multi-world operation)
(void)patch_.boundaryMesh().mesh().tetBasePtIs();
const label myComm = getCommunicator(); // Get or create
//// Make sure if running in database that there is a syncObjects FO
@ -1095,6 +1101,9 @@ void Foam::mappedPatchBase::calcAMI() const
const label myComm = getCommunicator(); // Get or create
// Pre-calculate surface (if any)
const auto& surf = surfPtr();
const label oldWorldComm(Pstream::worldComm);
const label oldWarnComm(Pstream::warnComm);
@ -1136,7 +1145,7 @@ void Foam::mappedPatchBase::calcAMI() const
Pstream::worldComm = myComm;
Pstream::warnComm = Pstream::worldComm;
AMIPtr_->calculate(patch_, nbrPatch0, surfPtr());
AMIPtr_->calculate(patch_, nbrPatch0, surf);
Pstream::warnComm = oldWarnComm;
Pstream::worldComm = oldWorldComm;
@ -1160,14 +1169,14 @@ void Foam::mappedPatchBase::calcAMI() const
// Construct/apply AMI interpolation to determine addressing
// and weights. Have patch_ for src faces, 0 faces for the
// target side
AMIPtr_->calculate(patch_, dummyPatch, surfPtr());
AMIPtr_->calculate(patch_, dummyPatch, surf);
}
else
{
// Construct/apply AMI interpolation to determine addressing
// and weights. Have 0 faces for src side, patch_ for the tgt
// side
AMIPtr_->calculate(dummyPatch, patch_, surfPtr());
AMIPtr_->calculate(dummyPatch, patch_, surf);
}
// Now the AMI addressing/weights will be from src side (on masterWorld
// processors) to tgt side (on other processors)

83
src/meshTools/mappedPatches/mappedPolyPatch/mappedPatchBaseTemplates.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2016 OpenFOAM Foundation
Copyright (C) 2018-2021 OpenCFD Ltd.
Copyright (C) 2018-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -31,19 +31,21 @@ void Foam::mappedPatchBase::distribute(List<Type>& lst) const
{
const label myComm = getCommunicator(); // Get or create
const label oldWarnComm(Pstream::warnComm);
Pstream::warnComm = myComm;
switch (mode_)
{
case NEARESTPATCHFACEAMI:
{
const label oldWorldComm(Pstream::worldComm);
const auto& interp = AMI();
Pstream::warnComm = myComm;
Pstream::worldComm = myComm;
if (sameWorld())
{
// lst is the other side's values
lst = AMI().interpolateToSource(Field<Type>(std::move(lst)));
lst = interp.interpolateToSource(Field<Type>(std::move(lst)));
}
else
{
@ -58,9 +60,9 @@ void Foam::mappedPatchBase::distribute(List<Type>& lst) const
tmp<Field<Type>> tmasterFld
(
AMI().interpolateToSource(Field<Type>(0))
interp.interpolateToSource(Field<Type>(0))
);
(void)AMI().interpolateToTarget
(void)interp.interpolateToTarget
(
Field<Type>(std::move(lst))
);
@ -71,13 +73,13 @@ void Foam::mappedPatchBase::distribute(List<Type>& lst) const
}
else
{
(void)AMI().interpolateToSource
(void)interp.interpolateToSource
(
Field<Type>(std::move(lst))
);
tmp<Field<Type>> tmasterFld
(
AMI().interpolateToTarget(Field<Type>(0))
interp.interpolateToTarget(Field<Type>(0))
);
// We've received in our interpolateToTarget the
@ -86,15 +88,18 @@ void Foam::mappedPatchBase::distribute(List<Type>& lst) const
}
}
Pstream::worldComm = oldWorldComm;
Pstream::warnComm = oldWarnComm;
break;
}
default:
{
map().distribute(lst);
const auto& m = map();
Pstream::warnComm = m.comm();
m.distribute(lst);
Pstream::warnComm = oldWarnComm;
}
}
Pstream::warnComm = oldWarnComm;
}
@ -107,28 +112,35 @@ void Foam::mappedPatchBase::distribute
{
const label myComm = getCommunicator(); // Get or create
const label oldWarnComm(Pstream::warnComm);
Pstream::warnComm = myComm;
switch (mode_)
{
case NEARESTPATCHFACEAMI:
{
const label oldWorldComm(Pstream::worldComm);
const auto& interp = AMI();
Pstream::warnComm = myComm;
Pstream::worldComm = myComm;
lst = AMI().interpolateToSource(Field<Type>(std::move(lst)), cop);
lst = interp.interpolateToSource(Field<Type>(std::move(lst)), cop);
Pstream::worldComm = oldWorldComm;
Pstream::warnComm = oldWarnComm;
break;
}
default:
{
// Force early construction of parallel data
(void)patch_.boundaryMesh().mesh().tetBasePtIs();
const auto& m = map();
Pstream::warnComm = myComm;
mapDistributeBase::distribute
(
Pstream::defaultCommsType,
map().schedule(),
map().constructSize(),
map().subMap(),
m.schedule(),
m.constructSize(),
m.subMap(),
false,
map().constructMap(),
m.constructMap(),
false,
lst,
Type(Zero),
@ -137,10 +149,9 @@ void Foam::mappedPatchBase::distribute
UPstream::msgType(),
myComm
);
Pstream::warnComm = oldWarnComm;
}
}
Pstream::warnComm = oldWarnComm;
}
@ -149,26 +160,32 @@ void Foam::mappedPatchBase::reverseDistribute(List<Type>& lst) const
{
const label myComm = getCommunicator(); // Get or create
const label oldWarnComm(Pstream::warnComm);
Pstream::warnComm = myComm;
switch (mode_)
{
case NEARESTPATCHFACEAMI:
{
const label oldWorldComm(Pstream::worldComm);
const auto& interp = AMI();
Pstream::warnComm = myComm;
Pstream::worldComm = myComm;
lst = AMI().interpolateToTarget(Field<Type>(std::move(lst)));
lst = interp.interpolateToTarget(Field<Type>(std::move(lst)));
Pstream::worldComm = oldWorldComm;
Pstream::warnComm = oldWarnComm;
break;
}
default:
{
map().reverseDistribute(sampleSize(), lst);
// Force early construction of parallel data
(void)patch_.boundaryMesh().mesh().tetBasePtIs();
const auto& m = map();
Pstream::warnComm = m.comm();
m.reverseDistribute(sampleSize(), lst);
Pstream::warnComm = oldWarnComm;
break;
}
}
Pstream::warnComm = oldWarnComm;
}
@ -181,29 +198,34 @@ void Foam::mappedPatchBase::reverseDistribute
{
const label myComm = getCommunicator(); // Get or create
const label oldWarnComm(Pstream::warnComm);
Pstream::warnComm = myComm;
switch (mode_)
{
case NEARESTPATCHFACEAMI:
{
const label oldWorldComm(Pstream::worldComm);
const auto& interp = AMI();
Pstream::warnComm = myComm;
Pstream::worldComm = myComm;
lst = AMI().interpolateToTarget(Field<Type>(std::move(lst)), cop);
lst = interp.interpolateToTarget(Field<Type>(std::move(lst)), cop);
Pstream::worldComm = oldWorldComm;
Pstream::warnComm = oldWarnComm;
break;
}
default:
{
label cSize = sampleSize();
(void)patch_.boundaryMesh().mesh().tetBasePtIs();
const auto& m = map();
const label cSize = sampleSize();
Pstream::warnComm = myComm;
mapDistributeBase::distribute
(
Pstream::defaultCommsType,
map().schedule(),
m.schedule(),
cSize,
map().constructMap(),
m.constructMap(),
false,
map().subMap(),
m.subMap(),
false,
lst,
Type(Zero),
@ -212,11 +234,10 @@ void Foam::mappedPatchBase::reverseDistribute
UPstream::msgType(),
myComm
);
Pstream::warnComm = oldWarnComm;
break;
}
}
Pstream::warnComm = oldWarnComm;
}