From 873f7aac2d573f36dea6ba29a7f6d43676651a8a Mon Sep 17 00:00:00 2001 From: mattijs Date: Wed, 20 Jul 2022 13:34:58 +0100 Subject: [PATCH] BUG: multi-world: avoid on-the-fly comms. Fixes #2529 Probably can get away with less tetBasePtIs() triggering --- .../mappedPolyPatch/mappedPatchBase.C | 15 +++- .../mappedPatchBaseTemplates.C | 83 ++++++++++++------- 2 files changed, 64 insertions(+), 34 deletions(-) diff --git a/src/meshTools/mappedPatches/mappedPolyPatch/mappedPatchBase.C b/src/meshTools/mappedPatches/mappedPolyPatch/mappedPatchBase.C index 29a590e6e5..a1ea79bf9e 100644 --- a/src/meshTools/mappedPatches/mappedPolyPatch/mappedPatchBase.C +++ b/src/meshTools/mappedPatches/mappedPolyPatch/mappedPatchBase.C @@ -733,6 +733,12 @@ void Foam::mappedPatchBase::calcMapping() const DebugInFunction << nl; + + // Early construction of tetBasePtIs since does parallel + // communication which might conflict with inter-world communicator + // (for multi-world operation) + (void)patch_.boundaryMesh().mesh().tetBasePtIs(); + const label myComm = getCommunicator(); // Get or create //// Make sure if running in database that there is a syncObjects FO @@ -1095,6 +1101,9 @@ void Foam::mappedPatchBase::calcAMI() const const label myComm = getCommunicator(); // Get or create + // Pre-calculate surface (if any) + const auto& surf = surfPtr(); + const label oldWorldComm(Pstream::worldComm); const label oldWarnComm(Pstream::warnComm); @@ -1136,7 +1145,7 @@ void Foam::mappedPatchBase::calcAMI() const Pstream::worldComm = myComm; Pstream::warnComm = Pstream::worldComm; - AMIPtr_->calculate(patch_, nbrPatch0, surfPtr()); + AMIPtr_->calculate(patch_, nbrPatch0, surf); Pstream::warnComm = oldWarnComm; Pstream::worldComm = oldWorldComm; @@ -1160,14 +1169,14 @@ void Foam::mappedPatchBase::calcAMI() const // Construct/apply AMI interpolation to determine addressing // and weights. Have patch_ for src faces, 0 faces for the // target side - AMIPtr_->calculate(patch_, dummyPatch, surfPtr()); + AMIPtr_->calculate(patch_, dummyPatch, surf); } else { // Construct/apply AMI interpolation to determine addressing // and weights. Have 0 faces for src side, patch_ for the tgt // side - AMIPtr_->calculate(dummyPatch, patch_, surfPtr()); + AMIPtr_->calculate(dummyPatch, patch_, surf); } // Now the AMI addressing/weights will be from src side (on masterWorld // processors) to tgt side (on other processors) diff --git a/src/meshTools/mappedPatches/mappedPolyPatch/mappedPatchBaseTemplates.C b/src/meshTools/mappedPatches/mappedPolyPatch/mappedPatchBaseTemplates.C index 130f10823e..7e93773202 100644 --- a/src/meshTools/mappedPatches/mappedPolyPatch/mappedPatchBaseTemplates.C +++ b/src/meshTools/mappedPatches/mappedPolyPatch/mappedPatchBaseTemplates.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2011-2016 OpenFOAM Foundation - Copyright (C) 2018-2021 OpenCFD Ltd. + Copyright (C) 2018-2022 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -31,19 +31,21 @@ void Foam::mappedPatchBase::distribute(List& lst) const { const label myComm = getCommunicator(); // Get or create const label oldWarnComm(Pstream::warnComm); - Pstream::warnComm = myComm; switch (mode_) { case NEARESTPATCHFACEAMI: { const label oldWorldComm(Pstream::worldComm); + const auto& interp = AMI(); + + Pstream::warnComm = myComm; Pstream::worldComm = myComm; if (sameWorld()) { // lst is the other side's values - lst = AMI().interpolateToSource(Field(std::move(lst))); + lst = interp.interpolateToSource(Field(std::move(lst))); } else { @@ -58,9 +60,9 @@ void Foam::mappedPatchBase::distribute(List& lst) const tmp> tmasterFld ( - AMI().interpolateToSource(Field(0)) + interp.interpolateToSource(Field(0)) ); - (void)AMI().interpolateToTarget + (void)interp.interpolateToTarget ( Field(std::move(lst)) ); @@ -71,13 +73,13 @@ void Foam::mappedPatchBase::distribute(List& lst) const } else { - (void)AMI().interpolateToSource + (void)interp.interpolateToSource ( Field(std::move(lst)) ); tmp> tmasterFld ( - AMI().interpolateToTarget(Field(0)) + interp.interpolateToTarget(Field(0)) ); // We've received in our interpolateToTarget the @@ -86,15 +88,18 @@ void Foam::mappedPatchBase::distribute(List& lst) const } } Pstream::worldComm = oldWorldComm; + Pstream::warnComm = oldWarnComm; break; } default: { - map().distribute(lst); + const auto& m = map(); + + Pstream::warnComm = m.comm(); + m.distribute(lst); + Pstream::warnComm = oldWarnComm; } } - - Pstream::warnComm = oldWarnComm; } @@ -107,28 +112,35 @@ void Foam::mappedPatchBase::distribute { const label myComm = getCommunicator(); // Get or create const label oldWarnComm(Pstream::warnComm); - Pstream::warnComm = myComm; switch (mode_) { case NEARESTPATCHFACEAMI: { const label oldWorldComm(Pstream::worldComm); + const auto& interp = AMI(); + Pstream::warnComm = myComm; Pstream::worldComm = myComm; - lst = AMI().interpolateToSource(Field(std::move(lst)), cop); + lst = interp.interpolateToSource(Field(std::move(lst)), cop); Pstream::worldComm = oldWorldComm; + Pstream::warnComm = oldWarnComm; break; } default: { + // Force early construction of parallel data + (void)patch_.boundaryMesh().mesh().tetBasePtIs(); + const auto& m = map(); + + Pstream::warnComm = myComm; mapDistributeBase::distribute ( Pstream::defaultCommsType, - map().schedule(), - map().constructSize(), - map().subMap(), + m.schedule(), + m.constructSize(), + m.subMap(), false, - map().constructMap(), + m.constructMap(), false, lst, Type(Zero), @@ -137,10 +149,9 @@ void Foam::mappedPatchBase::distribute UPstream::msgType(), myComm ); + Pstream::warnComm = oldWarnComm; } } - - Pstream::warnComm = oldWarnComm; } @@ -149,26 +160,32 @@ void Foam::mappedPatchBase::reverseDistribute(List& lst) const { const label myComm = getCommunicator(); // Get or create const label oldWarnComm(Pstream::warnComm); - Pstream::warnComm = myComm; switch (mode_) { case NEARESTPATCHFACEAMI: { const label oldWorldComm(Pstream::worldComm); + const auto& interp = AMI(); + Pstream::warnComm = myComm; Pstream::worldComm = myComm; - lst = AMI().interpolateToTarget(Field(std::move(lst))); + lst = interp.interpolateToTarget(Field(std::move(lst))); Pstream::worldComm = oldWorldComm; + Pstream::warnComm = oldWarnComm; break; } default: { - map().reverseDistribute(sampleSize(), lst); + // Force early construction of parallel data + (void)patch_.boundaryMesh().mesh().tetBasePtIs(); + const auto& m = map(); + + Pstream::warnComm = m.comm(); + m.reverseDistribute(sampleSize(), lst); + Pstream::warnComm = oldWarnComm; break; } } - - Pstream::warnComm = oldWarnComm; } @@ -181,29 +198,34 @@ void Foam::mappedPatchBase::reverseDistribute { const label myComm = getCommunicator(); // Get or create const label oldWarnComm(Pstream::warnComm); - Pstream::warnComm = myComm; switch (mode_) { case NEARESTPATCHFACEAMI: { const label oldWorldComm(Pstream::worldComm); + const auto& interp = AMI(); + Pstream::warnComm = myComm; Pstream::worldComm = myComm; - lst = AMI().interpolateToTarget(Field(std::move(lst)), cop); + lst = interp.interpolateToTarget(Field(std::move(lst)), cop); Pstream::worldComm = oldWorldComm; + Pstream::warnComm = oldWarnComm; break; } default: { - label cSize = sampleSize(); + (void)patch_.boundaryMesh().mesh().tetBasePtIs(); + const auto& m = map(); + const label cSize = sampleSize(); + Pstream::warnComm = myComm; mapDistributeBase::distribute ( Pstream::defaultCommsType, - map().schedule(), + m.schedule(), cSize, - map().constructMap(), + m.constructMap(), false, - map().subMap(), + m.subMap(), false, lst, Type(Zero), @@ -212,11 +234,10 @@ void Foam::mappedPatchBase::reverseDistribute UPstream::msgType(), myComm ); + Pstream::warnComm = oldWarnComm; break; } } - - Pstream::warnComm = oldWarnComm; }