zhyang-dev/openfoam
1
0
Fork 0
mirror of https://develop.openfoam.com/Development/openfoam.git synced 2025-11-28 03:28:01 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

BUG: multi-world: avoid on-the-fly comms. Fixes #2529

Browse Source 
Probably can get away with less tetBasePtIs() triggering
This commit is contained in:
mattijs
2022-07-20 13:34:58 +01:00
parent 66ddf0a104
commit 873f7aac2d
2 changed files with 64 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/meshTools/mappedPatches/mappedPolyPatch/mappedPatchBase.C
View File

@ -733,6 +733,12 @@ void Foam::mappedPatchBase::calcMapping() const
DebugInFunction << nl; DebugInFunction << nl;
// Early construction of tetBasePtIs since does parallel
// communication which might conflict with inter-world communicator
// (for multi-world operation)
(void)patch_.boundaryMesh().mesh().tetBasePtIs();
const label myComm = getCommunicator(); // Get or create const label myComm = getCommunicator(); // Get or create
//// Make sure if running in database that there is a syncObjects FO //// Make sure if running in database that there is a syncObjects FO
@ -1095,6 +1101,9 @@ void Foam::mappedPatchBase::calcAMI() const
const label myComm = getCommunicator(); // Get or create const label myComm = getCommunicator(); // Get or create
// Pre-calculate surface (if any)
const auto& surf = surfPtr();
const label oldWorldComm(Pstream::worldComm); const label oldWorldComm(Pstream::worldComm);
const label oldWarnComm(Pstream::warnComm); const label oldWarnComm(Pstream::warnComm);
@ -1136,7 +1145,7 @@ void Foam::mappedPatchBase::calcAMI() const
Pstream::worldComm = myComm; Pstream::worldComm = myComm;
Pstream::warnComm = Pstream::worldComm; Pstream::warnComm = Pstream::worldComm;
AMIPtr_->calculate(patch_, nbrPatch0, surfPtr()); AMIPtr_->calculate(patch_, nbrPatch0, surf);
Pstream::warnComm = oldWarnComm; Pstream::warnComm = oldWarnComm;
Pstream::worldComm = oldWorldComm; Pstream::worldComm = oldWorldComm;
@ -1160,14 +1169,14 @@ void Foam::mappedPatchBase::calcAMI() const
// Construct/apply AMI interpolation to determine addressing // Construct/apply AMI interpolation to determine addressing
// and weights. Have patch_ for src faces, 0 faces for the // and weights. Have patch_ for src faces, 0 faces for the
// target side // target side
AMIPtr_->calculate(patch_, dummyPatch, surfPtr()); AMIPtr_->calculate(patch_, dummyPatch, surf);
} }
else else
{ {
// Construct/apply AMI interpolation to determine addressing // Construct/apply AMI interpolation to determine addressing
// and weights. Have 0 faces for src side, patch_ for the tgt // and weights. Have 0 faces for src side, patch_ for the tgt
// side // side
AMIPtr_->calculate(dummyPatch, patch_, surfPtr()); AMIPtr_->calculate(dummyPatch, patch_, surf);
} }
// Now the AMI addressing/weights will be from src side (on masterWorld // Now the AMI addressing/weights will be from src side (on masterWorld
// processors) to tgt side (on other processors) // processors) to tgt side (on other processors)

85
src/meshTools/mappedPatches/mappedPolyPatch/mappedPatchBaseTemplates.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2011-2016 OpenFOAM Foundation Copyright (C) 2011-2016 OpenFOAM Foundation
Copyright (C) 2018-2021 OpenCFD Ltd. Copyright (C) 2018-2022 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
@ -31,19 +31,21 @@ void Foam::mappedPatchBase::distribute(List<Type>& lst) const
{ {
const label myComm = getCommunicator(); // Get or create const label myComm = getCommunicator(); // Get or create
const label oldWarnComm(Pstream::warnComm); const label oldWarnComm(Pstream::warnComm);
Pstream::warnComm = myComm;
switch (mode_) switch (mode_)
{ {
case NEARESTPATCHFACEAMI: case NEARESTPATCHFACEAMI:
{ {
const label oldWorldComm(Pstream::worldComm); const label oldWorldComm(Pstream::worldComm);
const auto& interp = AMI();
Pstream::warnComm = myComm;
Pstream::worldComm = myComm; Pstream::worldComm = myComm;
if (sameWorld()) if (sameWorld())
{ {
// lst is the other side's values // lst is the other side's values
lst = AMI().interpolateToSource(Field<Type>(std::move(lst))); lst = interp.interpolateToSource(Field<Type>(std::move(lst)));
} }
else else
{ {
@ -58,9 +60,9 @@ void Foam::mappedPatchBase::distribute(List<Type>& lst) const
tmp<Field<Type>> tmasterFld tmp<Field<Type>> tmasterFld
( (
AMI().interpolateToSource(Field<Type>(0)) interp.interpolateToSource(Field<Type>(0))
); );
(void)AMI().interpolateToTarget (void)interp.interpolateToTarget
( (
Field<Type>(std::move(lst)) Field<Type>(std::move(lst))
); );
@ -71,13 +73,13 @@ void Foam::mappedPatchBase::distribute(List<Type>& lst) const
} }
else else
{ {
(void)AMI().interpolateToSource (void)interp.interpolateToSource
( (
Field<Type>(std::move(lst)) Field<Type>(std::move(lst))
); );
tmp<Field<Type>> tmasterFld tmp<Field<Type>> tmasterFld
( (
AMI().interpolateToTarget(Field<Type>(0)) interp.interpolateToTarget(Field<Type>(0))
); );
// We've received in our interpolateToTarget the // We've received in our interpolateToTarget the
@ -86,16 +88,19 @@ void Foam::mappedPatchBase::distribute(List<Type>& lst) const
} }
} }
Pstream::worldComm = oldWorldComm; Pstream::worldComm = oldWorldComm;
Pstream::warnComm = oldWarnComm;
break; break;
} }
default: default:
{ {
map().distribute(lst); const auto& m = map();
}
}
Pstream::warnComm = m.comm();
m.distribute(lst);
Pstream::warnComm = oldWarnComm; Pstream::warnComm = oldWarnComm;
} }
}
}
template<class Type, class CombineOp> template<class Type, class CombineOp>
@ -107,28 +112,35 @@ void Foam::mappedPatchBase::distribute
{ {
const label myComm = getCommunicator(); // Get or create const label myComm = getCommunicator(); // Get or create
const label oldWarnComm(Pstream::warnComm); const label oldWarnComm(Pstream::warnComm);
Pstream::warnComm = myComm;
switch (mode_) switch (mode_)
{ {
case NEARESTPATCHFACEAMI: case NEARESTPATCHFACEAMI:
{ {
const label oldWorldComm(Pstream::worldComm); const label oldWorldComm(Pstream::worldComm);
const auto& interp = AMI();
Pstream::warnComm = myComm;
Pstream::worldComm = myComm; Pstream::worldComm = myComm;
lst = AMI().interpolateToSource(Field<Type>(std::move(lst)), cop); lst = interp.interpolateToSource(Field<Type>(std::move(lst)), cop);
Pstream::worldComm = oldWorldComm; Pstream::worldComm = oldWorldComm;
Pstream::warnComm = oldWarnComm;
break; break;
} }
default: default:
{ {
// Force early construction of parallel data
(void)patch_.boundaryMesh().mesh().tetBasePtIs();
const auto& m = map();
Pstream::warnComm = myComm;
mapDistributeBase::distribute mapDistributeBase::distribute
( (
Pstream::defaultCommsType, Pstream::defaultCommsType,
map().schedule(), m.schedule(),
map().constructSize(), m.constructSize(),
map().subMap(), m.subMap(),
false, false,
map().constructMap(), m.constructMap(),
false, false,
lst, lst,
Type(Zero), Type(Zero),
@ -137,11 +149,10 @@ void Foam::mappedPatchBase::distribute
UPstream::msgType(), UPstream::msgType(),
myComm myComm
); );
}
}
Pstream::warnComm = oldWarnComm; Pstream::warnComm = oldWarnComm;
} }
}
}
template<class Type> template<class Type>
@ -149,26 +160,32 @@ void Foam::mappedPatchBase::reverseDistribute(List<Type>& lst) const
{ {
const label myComm = getCommunicator(); // Get or create const label myComm = getCommunicator(); // Get or create
const label oldWarnComm(Pstream::warnComm); const label oldWarnComm(Pstream::warnComm);
Pstream::warnComm = myComm;
switch (mode_) switch (mode_)
{ {
case NEARESTPATCHFACEAMI: case NEARESTPATCHFACEAMI:
{ {
const label oldWorldComm(Pstream::worldComm); const label oldWorldComm(Pstream::worldComm);
const auto& interp = AMI();
Pstream::warnComm = myComm;
Pstream::worldComm = myComm; Pstream::worldComm = myComm;
lst = AMI().interpolateToTarget(Field<Type>(std::move(lst))); lst = interp.interpolateToTarget(Field<Type>(std::move(lst)));
Pstream::worldComm = oldWorldComm; Pstream::worldComm = oldWorldComm;
Pstream::warnComm = oldWarnComm;
break; break;
} }
default: default:
{ {
map().reverseDistribute(sampleSize(), lst); // Force early construction of parallel data
(void)patch_.boundaryMesh().mesh().tetBasePtIs();
const auto& m = map();
Pstream::warnComm = m.comm();
m.reverseDistribute(sampleSize(), lst);
Pstream::warnComm = oldWarnComm;
break; break;
} }
} }
Pstream::warnComm = oldWarnComm;
} }
@ -181,29 +198,34 @@ void Foam::mappedPatchBase::reverseDistribute
{ {
const label myComm = getCommunicator(); // Get or create const label myComm = getCommunicator(); // Get or create
const label oldWarnComm(Pstream::warnComm); const label oldWarnComm(Pstream::warnComm);
Pstream::warnComm = myComm;
switch (mode_) switch (mode_)
{ {
case NEARESTPATCHFACEAMI: case NEARESTPATCHFACEAMI:
{ {
const label oldWorldComm(Pstream::worldComm); const label oldWorldComm(Pstream::worldComm);
const auto& interp = AMI();
Pstream::warnComm = myComm;
Pstream::worldComm = myComm; Pstream::worldComm = myComm;
lst = AMI().interpolateToTarget(Field<Type>(std::move(lst)), cop); lst = interp.interpolateToTarget(Field<Type>(std::move(lst)), cop);
Pstream::worldComm = oldWorldComm; Pstream::worldComm = oldWorldComm;
Pstream::warnComm = oldWarnComm;
break; break;
} }
default: default:
{ {
label cSize = sampleSize(); (void)patch_.boundaryMesh().mesh().tetBasePtIs();
const auto& m = map();
const label cSize = sampleSize();
Pstream::warnComm = myComm;
mapDistributeBase::distribute mapDistributeBase::distribute
( (
Pstream::defaultCommsType, Pstream::defaultCommsType,
map().schedule(), m.schedule(),
cSize, cSize,
map().constructMap(), m.constructMap(),
false, false,
map().subMap(), m.subMap(),
false, false,
lst, lst,
Type(Zero), Type(Zero),
@ -212,11 +234,10 @@ void Foam::mappedPatchBase::reverseDistribute
UPstream::msgType(), UPstream::msgType(),
myComm myComm
); );
Pstream::warnComm = oldWarnComm;
break; break;
} }
} }
Pstream::warnComm = oldWarnComm;
} }