From d5422239f3ee6f100f939ea84c5e415ed1715447 Mon Sep 17 00:00:00 2001 From: Andrew Heather Date: Tue, 19 Jun 2018 12:00:25 +0100 Subject: [PATCH 1/4] ENH: Added new processor level of detail (LOD) to optimise parallel data transfers --- src/meshTools/Make/files | 5 + src/meshTools/processorLOD/box/box.C | 635 ++++++++++++++++++ src/meshTools/processorLOD/box/box.H | 206 ++++++ src/meshTools/processorLOD/cellBox/cellBox.C | 110 +++ src/meshTools/processorLOD/cellBox/cellBox.H | 120 ++++ src/meshTools/processorLOD/faceBox/faceBox.C | 82 +++ src/meshTools/processorLOD/faceBox/faceBox.H | 117 ++++ .../processorLOD/processorLOD/processorLOD.C | 46 ++ .../processorLOD/processorLOD/processorLOD.H | 97 +++ 9 files changed, 1418 insertions(+) create mode 100644 src/meshTools/processorLOD/box/box.C create mode 100644 src/meshTools/processorLOD/box/box.H create mode 100644 src/meshTools/processorLOD/cellBox/cellBox.C create mode 100644 src/meshTools/processorLOD/cellBox/cellBox.H create mode 100644 src/meshTools/processorLOD/faceBox/faceBox.C create mode 100644 src/meshTools/processorLOD/faceBox/faceBox.H create mode 100644 src/meshTools/processorLOD/processorLOD/processorLOD.C create mode 100644 src/meshTools/processorLOD/processorLOD/processorLOD.H diff --git a/src/meshTools/Make/files b/src/meshTools/Make/files index 45f839d1a0..56dc286f3a 100644 --- a/src/meshTools/Make/files +++ b/src/meshTools/Make/files @@ -231,6 +231,11 @@ triSurface/triSurfaceTools/pointToPointPlanarInterpolation.C twoDPointCorrector/twoDPointCorrector.C +processorLOD/processorLOD/processorLOD.C +processorLOD/box/box.C +processorLOD/cellBox/cellBox.C +processorLOD/faceBox/faceBox.C + AMI=AMIInterpolation $(AMI)/AMIInterpolation/AMIInterpolationName.C $(AMI)/AMIInterpolation/AMIPatchToPatchInterpolation.C diff --git a/src/meshTools/processorLOD/box/box.C b/src/meshTools/processorLOD/box/box.C new file mode 100644 index 0000000000..f176ab0fd0 --- /dev/null +++ b/src/meshTools/processorLOD/box/box.C @@ -0,0 +1,635 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | OpenFOAM: The Open Source CFD Toolbox + \\ / O peration | + \\ / A nd | Copyright (C) 2017 OpenCFD Ltd. + \\/ M anipulation | +------------------------------------------------------------------------------- +License + This file is part of OpenFOAM. + + OpenFOAM is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with OpenFOAM. If not, see . + +\*---------------------------------------------------------------------------*/ + +#include "box.H" +#include "mapDistribute.H" +#include "OFstream.H" +#include "meshTools.H" + +const Foam::label Foam::processorLODs::box::DROP = 0; +const Foam::label Foam::processorLODs::box::REFINE = 1; +const Foam::label Foam::processorLODs::box::FIXED = 2; +const Foam::label Foam::processorLODs::box::nStartUpIter = 2; + +namespace Foam +{ +namespace processorLODs +{ + defineTypeNameAndDebug(box, 0); +} +} + +// * * * * * * * * * * * * Protected Member Functions * * * * * * * * * * * // + +void Foam::processorLODs::box::writeBoxes +( + const List>& fixedBoxes, + const label iter +) const +{ + static label time = 0; + + OFstream os + ( + "processor" + Foam::name(Pstream::myProcNo()) + + "_time" + Foam::name(time) + + "_iter" + Foam::name(iter) + ".obj" + ); + + label verti = 0; + for (label proci = 0; proci < Pstream::nProcs(); ++proci) + { + if (proci == Pstream::myProcNo()) + { + continue; + } + + const DynamicList& procBoxes = fixedBoxes[proci]; + forAll(procBoxes, boxi) + { + const treeBoundBox& bb = procBoxes[boxi]; + + // Write the points + const pointField pts(bb.points()); + for (const point& p : pts) + { + meshTools::writeOBJ(os, p); + } + + // Write the box faces + for (const face& f : bb.faces) + { + os << 'f'; + for (const label fpi : f) + { + os << ' ' << fpi + verti + 1; + } + os << nl; + } + verti += pts.size(); + } + } + + ++time; +} + + +void Foam::processorLODs::box::setRefineFlags +( + const label refineIter, + const label nTgtObjects, + List& fixedSendElems, + List>& localTgtElems, + List& refineFlags, + labelList& nElems +) const +{ + PstreamBuffers pBufs(Pstream::commsTypes::nonBlocking); + + // Identify src boxes that can be refined and send to all remote procs + for (label proci = 0; proci < Pstream::nProcs(); ++proci) + { + if (proci != Pstream::myProcNo()) + { + UOPstream toProc(proci, pBufs); + toProc << nObjectsOfType_ << boxes_[proci] << newToOld_[proci]; + } + } + + pBufs.finishedSends(); + + // Test each remote src bb with local tgt objects to identify which remote + // src boxes can/should be refined + for (label proci = 0; proci < Pstream::nProcs(); ++proci) + { + if (proci == Pstream::myProcNo()) + { + // Not refining boxes I send to myself - will be sending all local + // elements + continue; + } + + // Receive the subset of changing src bound boxes for proci + UIPstream fromProc(proci, pBufs); + const label nObjects = readLabel(fromProc); + List remoteSrcBoxes(fromProc); + const List