From 71ccf51ba59a438410a71bfd456841891acceaae Mon Sep 17 00:00:00 2001 From: Will Bainbridge Date: Thu, 20 Jul 2023 11:42:18 +0100 Subject: [PATCH] decomposePar, reconstructPar: Do all regions simultaneously DecomposePar and reconstructPar now interleave the processing of multiple regions. This means that with the -allRegions option, the earlier times are completed in their entirety before later times are considered. It also lets regions to access each other during decomposition and reconstruction, which will be important for non-conformal region interfaces. To aid interpretation of the log, region prefixing is now used by both utilities in the same way as is done by foamMultiRun. DecomposePar has been overhauled so that it matches reconstructPar much more closely, both in terms of output and of iteration sequence. All meshes and addressing are loaded simultaneously and each field is considered in turn. Previously, all the fields were loaded, and each process and addressing set was considered in turn. This new strategy optimises memory usage for cases with lots of fields. --- .../decomposePar/Make/files | 1 - .../decomposePar/decomposePar.C | 1143 ++++++----------- .../decomposePar/dimFieldDecomposer.C | 49 - .../decomposePar/dimFieldDecomposer.H | 121 -- .../dimFieldDecomposerDecomposeFields.C | 75 -- .../decomposePar/fvFieldDecomposer.C | 76 +- .../decomposePar/fvFieldDecomposer.H | 74 +- .../fvFieldDecomposerDecomposeFields.C | 364 ------ .../decomposePar/fvFieldDecomposerTemplates.C | 550 ++++++++ .../decomposePar/lagrangianFieldDecomposer.C | 200 ++- .../decomposePar/lagrangianFieldDecomposer.H | 111 +- ...lagrangianFieldDecomposerDecomposeFields.C | 213 --- .../lagrangianFieldDecomposerTemplates.C | 119 ++ .../decomposePar/pointFieldDecomposer.C | 62 +- .../decomposePar/pointFieldDecomposer.H | 46 +- .../pointFieldDecomposerDecomposeFields.C | 111 -- .../pointFieldDecomposerTemplates.C | 163 +++ .../decomposePar/readFields.C | 107 -- .../decomposePar/readFields.H | 75 -- .../reconstructPar/Make/files | 2 +- .../reconstructPar/fvFieldReconstructor.C | 27 +- .../reconstructPar/fvFieldReconstructor.H | 87 +- ...elds.C => fvFieldReconstructorTemplates.C} | 269 ++-- .../lagrangianFieldReconstructor.C | 124 ++ .../lagrangianFieldReconstructor.H | 165 +++ .../lagrangianFieldReconstructorTemplates.C | 179 +++ .../reconstructPar/pointFieldReconstructor.C | 25 +- .../reconstructPar/pointFieldReconstructor.H | 36 +- ...s.C => pointFieldReconstructorTemplates.C} | 48 +- .../reconstructPar/reconstructLagrangian.H | 117 -- .../reconstructLagrangianFields.C | 273 ---- .../reconstructLagrangianPositions.C | 83 -- .../reconstructPar/reconstructPar.C | 788 +++++------- src/parallel/parallel/Make/files | 3 + src/parallel/parallel/domainDecomposition.C | 743 +---------- src/parallel/parallel/domainDecomposition.H | 104 +- .../parallel/domainDecompositionDecompose.C | 57 +- .../domainDecompositionNonConformal.C | 744 +++++++++++ .../parallel/domainDecompositionReconstruct.C | 40 +- .../parallel/multiDomainDecomposition.C | 180 +++ .../parallel/multiDomainDecomposition.H | 156 +++ src/parallel/parallel/multiRegionPrefixer.C | 123 ++ src/parallel/parallel/multiRegionPrefixer.H | 241 ++++ 43 files changed, 4246 insertions(+), 4028 deletions(-) delete mode 100644 applications/utilities/parallelProcessing/decomposePar/dimFieldDecomposer.C delete mode 100644 applications/utilities/parallelProcessing/decomposePar/dimFieldDecomposer.H delete mode 100644 applications/utilities/parallelProcessing/decomposePar/dimFieldDecomposerDecomposeFields.C delete mode 100644 applications/utilities/parallelProcessing/decomposePar/fvFieldDecomposerDecomposeFields.C create mode 100644 applications/utilities/parallelProcessing/decomposePar/fvFieldDecomposerTemplates.C delete mode 100644 applications/utilities/parallelProcessing/decomposePar/lagrangianFieldDecomposerDecomposeFields.C create mode 100644 applications/utilities/parallelProcessing/decomposePar/lagrangianFieldDecomposerTemplates.C delete mode 100644 applications/utilities/parallelProcessing/decomposePar/pointFieldDecomposerDecomposeFields.C create mode 100644 applications/utilities/parallelProcessing/decomposePar/pointFieldDecomposerTemplates.C delete mode 100644 applications/utilities/parallelProcessing/decomposePar/readFields.C delete mode 100644 applications/utilities/parallelProcessing/decomposePar/readFields.H rename applications/utilities/parallelProcessing/reconstructPar/{fvFieldReconstructorReconstructFields.C => fvFieldReconstructorTemplates.C} (84%) create mode 100644 applications/utilities/parallelProcessing/reconstructPar/lagrangianFieldReconstructor.C create mode 100644 applications/utilities/parallelProcessing/reconstructPar/lagrangianFieldReconstructor.H create mode 100644 applications/utilities/parallelProcessing/reconstructPar/lagrangianFieldReconstructorTemplates.C rename applications/utilities/parallelProcessing/reconstructPar/{pointFieldReconstructorReconstructFields.C => pointFieldReconstructorTemplates.C} (87%) delete mode 100644 applications/utilities/parallelProcessing/reconstructPar/reconstructLagrangian.H delete mode 100644 applications/utilities/parallelProcessing/reconstructPar/reconstructLagrangianFields.C delete mode 100644 applications/utilities/parallelProcessing/reconstructPar/reconstructLagrangianPositions.C create mode 100644 src/parallel/parallel/domainDecompositionNonConformal.C create mode 100644 src/parallel/parallel/multiDomainDecomposition.C create mode 100644 src/parallel/parallel/multiDomainDecomposition.H create mode 100644 src/parallel/parallel/multiRegionPrefixer.C create mode 100644 src/parallel/parallel/multiRegionPrefixer.H diff --git a/applications/utilities/parallelProcessing/decomposePar/Make/files b/applications/utilities/parallelProcessing/decomposePar/Make/files index e5f8107a96..cbc2c6bf7e 100644 --- a/applications/utilities/parallelProcessing/decomposePar/Make/files +++ b/applications/utilities/parallelProcessing/decomposePar/Make/files @@ -1,5 +1,4 @@ decomposePar.C -dimFieldDecomposer.C fvFieldDecomposer.C pointFieldDecomposer.C lagrangianFieldDecomposer.C diff --git a/applications/utilities/parallelProcessing/decomposePar/decomposePar.C b/applications/utilities/parallelProcessing/decomposePar/decomposePar.C index 608c07a8b6..19db79a679 100644 --- a/applications/utilities/parallelProcessing/decomposePar/decomposePar.C +++ b/applications/utilities/parallelProcessing/decomposePar/decomposePar.C @@ -50,11 +50,10 @@ Usage Copy any \a uniform directories too. - \par -constant + Decompose mesh and fields in the constant directory. - \par -time xxx:yyy \n - Override controlDict settings and decompose selected times. Does not - re-decompose the mesh i.e. does not handle moving mesh or changing - mesh cases. + Override controlDict settings and decompose selected times. - \par -fields \n Use existing geometry decomposition and convert fields only. @@ -68,27 +67,12 @@ Usage \*---------------------------------------------------------------------------*/ -#include "processorRunTimes.H" -#include "domainDecomposition.H" -#include "decompositionMethod.H" #include "argList.H" #include "timeSelector.H" - -#include "labelIOField.H" -#include "labelFieldIOField.H" -#include "scalarIOField.H" -#include "scalarFieldIOField.H" -#include "vectorIOField.H" -#include "vectorFieldIOField.H" -#include "sphericalTensorIOField.H" -#include "sphericalTensorFieldIOField.H" -#include "symmTensorIOField.H" -#include "symmTensorFieldIOField.H" -#include "tensorIOField.H" -#include "tensorFieldIOField.H" - -#include "readFields.H" -#include "dimFieldDecomposer.H" +#include "IOobjectList.H" +#include "processorRunTimes.H" +#include "multiDomainDecomposition.H" +#include "decompositionMethod.H" #include "fvFieldDecomposer.H" #include "pointFieldDecomposer.H" #include "lagrangianFieldDecomposer.H" @@ -100,40 +84,53 @@ using namespace Foam; namespace Foam { +bool haveUniform +( + const processorRunTimes& runTimes, + const word& regionDir = word::null +) +{ + return + fileHandler().isDir + ( + runTimes.completeTime().timePath()/regionDir/"uniform" + ); +} + + void decomposeUniform ( const bool copyUniform, - const bool distributeUniform, - const Time& runTime, - const Time& procRunTime, + const processorRunTimes& runTimes, const word& regionDir = word::null ) { const fileName uniformDir(regionDir/"uniform"); - if (fileHandler().isDir(runTime.timePath()/uniformDir)) + forAll(runTimes.procTimes(), proci) { - Info<< "Detected additional non-decomposed files in " - << runTime.timePath()/uniformDir - << endl; + const fileName procTimePath = + fileHandler().filePath(runTimes.procTimes()[proci].timePath()); - const fileName timePath = - fileHandler().filePath(procRunTime.timePath()); - - if (copyUniform || distributeUniform) + if (!fileHandler().isDir(procTimePath)) { - if (!fileHandler().exists(timePath/uniformDir)) + fileHandler().mkDir(procTimePath); + } + + if (copyUniform) + { + if (!fileHandler().exists(procTimePath/uniformDir)) { fileHandler().cp ( - runTime.timePath()/uniformDir, - timePath/uniformDir + runTimes.completeTime().timePath()/uniformDir, + procTimePath/uniformDir ); } } else { - // link with relative paths + // Link with relative paths string parentPath = string("..")/".."; if (regionDir != word::null) @@ -142,16 +139,18 @@ void decomposeUniform } fileName currentDir(cwd()); - chDir(timePath); + + chDir(procTimePath); if (!fileHandler().exists(uniformDir)) { fileHandler().ln ( - parentPath/runTime.name()/uniformDir, + parentPath/runTimes.completeTime().name()/uniformDir, uniformDir ); } + chDir(currentDir); } } @@ -179,14 +178,41 @@ void writeDecomposition(const domainDecomposition& meshes) cellProc.write(); Info<< "Wrote decomposition as volScalarField::Internal to " - << cellProc.name() << " for use in postprocessing." - << nl << endl; + << cellProc.name() << " for use in postprocessing" + << endl; } } +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +namespace Foam +{ + +class delayedNewLine +{ + mutable bool first_; + +public: + + delayedNewLine() + : + first_(true) + {} + + friend Ostream& operator<<(Ostream& os, const delayedNewLine& dnl) + { + if (!dnl.first_) os << nl; + dnl.first_ = false; + return os; + } +}; + +} + + // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // int main(int argc, char *argv[]) @@ -203,7 +229,7 @@ int main(int argc, char *argv[]) ( "cellProc", "write cell processor indices as a volScalarField::Internal for " - "post-processing." + "post-processing" ); argList::addBoolOption ( @@ -236,19 +262,19 @@ int main(int argc, char *argv[]) "remove existing processor*/ subdirs before decomposing the geometry" ); - // Include explicit constant options, have zero from time range + // Include explicit constant option, execute from zero by default timeSelector::addOptions(true, false); #include "setRootCase.H" - bool region = args.optionFound("region"); - bool writeCellProc = args.optionFound("cellProc"); - bool copyZero = args.optionFound("copyZero"); - bool copyUniform = args.optionFound("copyUniform"); - bool decomposeFieldsOnly = args.optionFound("fields"); - bool decomposeGeomOnly = args.optionFound("noFields"); - bool decomposeSets = !args.optionFound("noSets"); - bool forceOverwrite = args.optionFound("force"); + const bool region = args.optionFound("region"); + const bool writeCellProc = args.optionFound("cellProc"); + const bool copyZero = args.optionFound("copyZero"); + const bool copyUniform = args.optionFound("copyUniform"); + const bool decomposeFieldsOnly = args.optionFound("fields"); + const bool decomposeGeomOnly = args.optionFound("noFields"); + const bool decomposeSets = !args.optionFound("noSets"); + const bool forceOverwrite = args.optionFound("force"); if (decomposeGeomOnly) { @@ -264,14 +290,13 @@ int main(int argc, char *argv[]) } // Set time from database - Info<< "Create time\n" << endl; + Info<< "Create time" << nl << endl; processorRunTimes runTimes(Foam::Time::controlDictName, args); + const Time& runTime = runTimes.completeTime(); // Allow override of time const instantList times = runTimes.selectComplete(args); - const Time& runTime = runTimes.completeTime(); - #include "setRegionNames.H" // Remove existing processor directories if requested @@ -288,7 +313,7 @@ int main(int argc, char *argv[]) fileHandler().nProcs(runTimes.completeTime().path()); Info<< "Removing " << nProcs0 - << " existing processor directories" << endl; + << " existing processor directories" << nl << endl; // Remove existing processor directories const fileNameList dirs @@ -356,17 +381,12 @@ int main(int argc, char *argv[]) const dictionary decomposeParDict = decompositionMethod::decomposeParDict(runTimes.completeTime()); - // Decompose all regions + // Check existing decomposition forAll(regionNames, regioni) { const word& regionName = regionNames[regioni]; - - const word& regionDir = - regionName == polyMesh::defaultRegion - ? word::null - : regionName; - - Info<< "\n\nDecomposing mesh " << regionName << nl << endl; + const word regionDir = + regionName == polyMesh::defaultRegion ? word::null : regionName; // Determine the existing processor count directly const label nProcs = @@ -388,690 +408,337 @@ int main(int argc, char *argv[]) << " domains as specified in decomposeParDict" << nl << exit(FatalError); } + } - // Get flag to determine whether or not to distribute uniform data - const label distributeUniform = - decomposeParDict.lookupOrDefault("distributed", false); + // Create meshes + multiDomainDecomposition regionMeshes(runTimes, regionNames); + if + ( + !(decomposeFieldsOnly && copyZero) + && regionMeshes.readDecompose(decomposeSets) + ) + { + Info<< endl; - // Create meshes - Info<< "Create mesh" << endl; - domainDecomposition meshes(runTimes, regionName); - if (!decomposeFieldsOnly || !copyZero) + if (writeCellProc) { - if (meshes.readDecompose(decomposeSets) && writeCellProc) + forAll(regionNames, regioni) { - writeDecomposition(meshes); - fileHandler().flush(); - } - } - - // Field maps. These are preserved if decomposing multiple times. - PtrList fieldDecomposerList - ( - meshes.nProcs() - ); - PtrList dimFieldDecomposerList - ( - meshes.nProcs() - ); - PtrList pointFieldDecomposerList - ( - meshes.nProcs() - ); - - // Loop over all times - forAll(times, timei) - { - // Set the time - runTimes.setTime(times[timei], timei); - - Info<< "Time = " << runTimes.completeTime().userTimeName() << endl; - - // Update the meshes, if necessary - fvMesh::readUpdateState state = fvMesh::UNCHANGED; - if (!decomposeFieldsOnly || !copyZero) - { - state = meshes.readUpdateDecompose(); - } - - // Write the mesh out, if necessary - if (decomposeFieldsOnly) - { - // Nothing to do - } - else if (state != fvMesh::UNCHANGED) - { - meshes.writeProcs(decomposeSets); - } - - // Write the decomposition, if necessary - if - ( - writeCellProc - && meshes.completeMesh().facesInstance() - == runTimes.completeTime().name() - ) - { - writeDecomposition(meshes); - fileHandler().flush(); - } - - // Clear the field maps if there has been topology change - if (state >= fvMesh::TOPO_CHANGE) - { - for (label proci = 0; proci < meshes.nProcs(); proci++) - { - fieldDecomposerList.set(proci, nullptr); - dimFieldDecomposerList.set(proci, nullptr); - pointFieldDecomposerList.set(proci, nullptr); - } - } - - // Decompose the fields at this time - if (decomposeGeomOnly) - { - // Do nothing - } - else if (copyZero) - { - // Copy the field files from the