From 3995456979ea6d67736b37ac93c2d27516242e59 Mon Sep 17 00:00:00 2001 From: Will Bainbridge Date: Thu, 10 Mar 2022 11:24:21 +0000 Subject: [PATCH] parallelProcessing: Various improvements boundaryProcAddressing has been removed. This has not been needed for a long time. decomposePar has been optimised for mininum IO, rather than minimum memory usage. decomposePar has also been corrected so that it can decompose sequences of time-varying meshes. --- .../manipulation/renumberMesh/renumberMesh.C | 33 +- .../splitMeshRegions/splitMeshRegions.C | 32 +- .../foamFormatConvert/foamFormatConvert.C | 8 +- .../decomposePar/Make/files | 2 - .../decomposePar/decomposePar.C | 616 +++---- .../decomposePar/domainDecomposition.C | 1426 +++++++++++++---- .../decomposePar/domainDecomposition.H | 177 +- .../domainDecompositionDistribute.C | 71 - .../decomposePar/domainDecompositionMesh.C | 482 ------ .../domainDecompositionTemplates.C | 125 -- .../decomposePar/fvFieldDecomposer.C | 76 +- .../decomposePar/fvFieldDecomposer.H | 20 +- .../fvFieldDecomposerDecomposeFields.C | 77 +- .../decomposePar/pointFieldDecomposer.C | 40 +- .../decomposePar/pointFieldDecomposer.H | 14 +- .../pointFieldDecomposerDecomposeFields.C | 12 +- .../reconstructPar/reconstructPar.C | 40 +- .../reconstructParMesh/reconstructParMesh.C | 45 +- .../reconstruct/fvFieldReconstructor.C | 9 +- .../reconstruct/fvFieldReconstructor.H | 8 +- .../fvFieldReconstructorReconstructFields.C | 16 +- .../reconstruct/pointFieldReconstructor.C | 11 +- .../reconstruct/pointFieldReconstructor.H | 8 +- ...pointFieldReconstructorReconstructFields.C | 9 +- .../reconstruct/reconstruct/processorMeshes.C | 23 +- .../reconstruct/reconstruct/processorMeshes.H | 10 +- 26 files changed, 1582 insertions(+), 1808 deletions(-) delete mode 100644 applications/utilities/parallelProcessing/decomposePar/domainDecompositionDistribute.C delete mode 100644 applications/utilities/parallelProcessing/decomposePar/domainDecompositionMesh.C delete mode 100644 applications/utilities/parallelProcessing/decomposePar/domainDecompositionTemplates.C diff --git a/applications/utilities/mesh/manipulation/renumberMesh/renumberMesh.C b/applications/utilities/mesh/manipulation/renumberMesh/renumberMesh.C index 70a8cb6070..7bb70c0471 100644 --- a/applications/utilities/mesh/manipulation/renumberMesh/renumberMesh.C +++ b/applications/utilities/mesh/manipulation/renumberMesh/renumberMesh.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | Website: https://openfoam.org - \\ / A nd | Copyright (C) 2011-2021 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2011-2022 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -799,18 +799,6 @@ int main(int argc, char *argv[]) ), labelList(0) ); - labelIOList boundaryProcAddressing - ( - IOobject - ( - "boundaryProcAddressing", - mesh.pointsInstance(), - polyMesh::meshSubDir, - mesh, - IOobject::READ_IF_PRESENT - ), - labelList(0) - ); // Read objects in time directory @@ -1246,25 +1234,6 @@ int main(int argc, char *argv[]) } } - if (boundaryProcAddressing.headerOk()) - { - boundaryProcAddressing.instance() = mesh.facesInstance(); - if (boundaryProcAddressing.size() == mesh.boundaryMesh().size()) - { - boundaryProcAddressing.write(); - } - else - { - const fileName fName(boundaryProcAddressing.filePath()); - if (fName.size()) - { - Info<< "Deleting inconsistent processor patch decomposition" - << " map " << fName << endl; - rm(fName); - } - } - } - if (writeMaps) { // For debugging: write out region diff --git a/applications/utilities/mesh/manipulation/splitMeshRegions/splitMeshRegions.C b/applications/utilities/mesh/manipulation/splitMeshRegions/splitMeshRegions.C index 6f6c29f9c5..ee8a8fa9a4 100644 --- a/applications/utilities/mesh/manipulation/splitMeshRegions/splitMeshRegions.C +++ b/applications/utilities/mesh/manipulation/splitMeshRegions/splitMeshRegions.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | Website: https://openfoam.org - \\ / A nd | Copyright (C) 2011-2021 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2011-2022 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -1004,36 +1004,6 @@ void createAndWriteRegion << " from region" << regioni << " cells back to base mesh." << endl; cellProcAddressing.write(); - - labelIOList boundaryProcAddressing - ( - IOobject - ( - "boundaryRegionAddressing", - newMesh().facesInstance(), - newMesh().meshSubDir, - newMesh(), - IOobject::NO_READ, - IOobject::NO_WRITE, - false - ), - labelList(nNewPatches, -1) - ); - forAll(oldToNew, i) - { - if (!addedPatches.found(i)) - { - label newI = oldToNew[i]; - if (newI >= 0 && newI < nNewPatches) - { - boundaryProcAddressing[oldToNew[i]] = i; - } - } - } - Info<< "Writing map " << boundaryProcAddressing.name() - << " from region" << regioni - << " boundary back to base mesh." << endl; - boundaryProcAddressing.write(); } diff --git a/applications/utilities/miscellaneous/foamFormatConvert/foamFormatConvert.C b/applications/utilities/miscellaneous/foamFormatConvert/foamFormatConvert.C index 8c08b996cd..0f0821fe06 100644 --- a/applications/utilities/miscellaneous/foamFormatConvert/foamFormatConvert.C +++ b/applications/utilities/miscellaneous/foamFormatConvert/foamFormatConvert.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | Website: https://openfoam.org - \\ / A nd | Copyright (C) 2011-2021 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2011-2022 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -293,12 +293,6 @@ int main(int argc, char *argv[]) writeMeshObject("pointProcAddressing", meshDir, runTime); writeMeshObject("faceProcAddressing", meshDir, runTime); writeMeshObject("cellProcAddressing", meshDir, runTime); - writeMeshObject - ( - "boundaryProcAddressing", - meshDir, - runTime - ); // foamyHexMesh vertices writeMeshObject diff --git a/applications/utilities/parallelProcessing/decomposePar/Make/files b/applications/utilities/parallelProcessing/decomposePar/Make/files index 125f6b81e7..077cbe902a 100644 --- a/applications/utilities/parallelProcessing/decomposePar/Make/files +++ b/applications/utilities/parallelProcessing/decomposePar/Make/files @@ -1,7 +1,5 @@ decomposePar.C domainDecomposition.C -domainDecompositionMesh.C -domainDecompositionDistribute.C dimFieldDecomposer.C fvFieldDecomposer.C pointFieldDecomposer.C diff --git a/applications/utilities/parallelProcessing/decomposePar/decomposePar.C b/applications/utilities/parallelProcessing/decomposePar/decomposePar.C index b558b547f7..734685e223 100644 --- a/applications/utilities/parallelProcessing/decomposePar/decomposePar.C +++ b/applications/utilities/parallelProcessing/decomposePar/decomposePar.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | Website: https://openfoam.org - \\ / A nd | Copyright (C) 2011-2021 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2011-2022 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -111,49 +111,15 @@ using namespace Foam; namespace Foam { -const labelIOList& procAddressing -( - const PtrList& procMeshList, - const label proci, - const word& name, - PtrList& procAddressingList -) -{ - const fvMesh& procMesh = procMeshList[proci]; - - if (!procAddressingList.set(proci)) - { - procAddressingList.set - ( - proci, - new labelIOList - ( - IOobject - ( - name, - procMesh.facesInstance(), - procMesh.meshSubDir, - procMesh, - IOobject::MUST_READ, - IOobject::NO_WRITE, - false - ) - ) - ); - } - return procAddressingList[proci]; -} - - void decomposeUniform ( const bool copyUniform, - const domainDecomposition& mesh, + const domainDecomposition& decomposition, const Time& processorDb, const word& regionDir = word::null ) { - const Time& runTime = mesh.time(); + const Time& runTime = decomposition.mesh().time(); // Any uniform data to copy/link? const fileName uniformDir(regionDir/"uniform"); @@ -167,7 +133,7 @@ void decomposeUniform const fileName timePath = fileHandler().filePath(processorDb.timePath()); - if (copyUniform || mesh.distributed()) + if (copyUniform || decomposition.distributed()) { if (!fileHandler().exists(timePath/uniformDir)) { @@ -204,6 +170,57 @@ void decomposeUniform } } + +void writeDecomposition(const domainDecomposition& decomposition) +{ + const labelList& procIds = decomposition.cellToProc(); + + // Write the decomposition as labelList for use with 'manual' + // decomposition method. + labelIOList cellDecomposition + ( + IOobject + ( + "cellDecomposition", + decomposition.mesh().facesInstance(), + decomposition.mesh(), + IOobject::NO_READ, + IOobject::NO_WRITE, + false + ), + procIds + ); + + cellDecomposition.write(); + + Info<< nl << "Wrote decomposition to " + << cellDecomposition.relativeObjectPath() + << " for use in manual decomposition." << endl; + + // Write as volScalarField for postprocessing. + volScalarField::Internal cellDist + ( + IOobject + ( + "cellDist", + decomposition.mesh().time().timeName(), + decomposition.mesh(), + IOobject::NO_READ, + IOobject::AUTO_WRITE + ), + decomposition.mesh(), + dimless, + scalarField(scalarList(procIds)) + ); + + cellDist.write(); + + Info<< nl << "Wrote decomposition as volScalarField to " + << cellDist.name() << " for use in postprocessing." + << endl; +} + + } @@ -291,17 +308,18 @@ int main(int argc, char *argv[]) } } - // Set time from database #include "createTime.H" // Allow override of time instantList times = timeSelector::selectIfPresent(runTime, args); + // Get region names const wordList regionNames(selectRegionNames(args, runTime)); + // Handle existing decomposition directories { - // Determine the existing processor count directly + // Determine the processor count from the directories label nProcs = fileHandler().nProcs(runTime.path()); if (forceOverwrite) @@ -367,7 +385,7 @@ int main(int argc, char *argv[]) } } - + // Decompose all regions forAll(regionNames, regioni) { const word& regionName = regionNames[regioni]; @@ -386,33 +404,25 @@ int main(int argc, char *argv[]) // Give file handler a chance to determine the output directory const_cast(fileHandler()).setNProcs(nDomains); - if (decomposeFieldsOnly) + // Sanity check number of processors in a previously decomposed case + if (decomposeFieldsOnly && nProcs != nDomains) { - // Sanity check on previously decomposed case - if (nProcs != nDomains) - { - FatalErrorInFunction - << "Specified -fields, but the case was decomposed with " - << nProcs << " domains" - << nl - << "instead of " << nDomains - << " domains as specified in decomposeParDict" - << nl - << exit(FatalError); - } + FatalErrorInFunction + << "Specified -fields, but the case was decomposed with " + << nProcs << " domains" << nl << "instead of " << nDomains + << " domains as specified in decomposeParDict" << nl + << exit(FatalError); } - else if (nProcs) + + // Reuse the decomposition if permitted + if (ifRequiredDecomposition && nProcs == nDomains) { - if (ifRequiredDecomposition && nProcs == nDomains) - { - // Reuse the decomposition - decomposeFieldsOnly = true; - Info<< "Using existing processor directories" << nl; - } + decomposeFieldsOnly = true; + Info<< "Using existing processor directories" << nl; } Info<< "Create mesh" << endl; - domainDecomposition mesh + fvMesh mesh ( IOobject ( @@ -422,148 +432,151 @@ int main(int argc, char *argv[]) IOobject::NO_READ, IOobject::NO_WRITE, false - ) + ), + false ); - // Decompose the mesh - if (!decomposeFieldsOnly) + // Create decomposition + domainDecomposition decomposition(mesh); + + // Read or generate a decomposition as necessary + if (decomposeFieldsOnly) { - mesh.decomposeMesh(); - - mesh.writeDecomposition(decomposeSets); - + decomposition.read(); + } + else + { + decomposition.decompose(); + decomposition.write(decomposeSets); if (writeCellDist) { - const labelList& procIds = mesh.cellToProc(); - - // Write the decomposition as labelList for use with 'manual' - // decomposition method. - labelIOList cellDecomposition - ( - IOobject - ( - "cellDecomposition", - mesh.facesInstance(), - mesh, - IOobject::NO_READ, - IOobject::NO_WRITE, - false - ), - procIds - ); - cellDecomposition.write(); - - Info<< nl << "Wrote decomposition to " - << cellDecomposition.relativeObjectPath() - << " for use in manual decomposition." << endl; - - // Write as volScalarField for postprocessing. - volScalarField cellDist - ( - IOobject - ( - "cellDist", - runTime.timeName(), - mesh, - IOobject::NO_READ, - IOobject::AUTO_WRITE - ), - mesh, - dimensionedScalar(dimless, 0) - ); - - forAll(procIds, celli) - { - cellDist[celli] = procIds[celli]; - } - - cellDist.write(); - - Info<< nl << "Wrote decomposition as volScalarField to " - << cellDist.name() << " for use in postprocessing." - << endl; + writeDecomposition(decomposition); } - fileHandler().flush(); } + // Field maps. These are preserved if decomposing multiple times. + PtrList fieldDecomposerList + ( + decomposition.nProcs() + ); + PtrList dimFieldDecomposerList + ( + decomposition.nProcs() + ); + PtrList pointFieldDecomposerList + ( + decomposition.nProcs() + ); - if (copyZero) + // Loop over all times + forAll(times, timeI) { - // Copy the 0 directory into each of the processor directories - fileName prevTimePath; - for (label proci = 0; proci < mesh.nProcs(); proci++) + // Set the time + runTime.setTime(times[timeI], timeI); + decomposition.setTime(times[timeI], timeI); + + Info<< "Time = " << runTime.userTimeName() << endl; + + // Update the mesh + const fvMesh::readUpdateState state = mesh.readUpdate(); + + // Update the decomposition + if (decomposeFieldsOnly) { - Time processorDb - ( - Time::controlDictName, - args.rootPath(), - args.caseName()/fileName(word("processor") + name(proci)) - ); - processorDb.setTime(runTime); - - if (fileHandler().isDir(runTime.timePath())) + decomposition.readUpdate(); + } + else if (state == fvMesh::POINTS_MOVED) + { + decomposition.writePoints(); + } + else if + ( + state == fvMesh::TOPO_CHANGE + || state == fvMesh::TOPO_PATCH_CHANGE + ) + { + decomposition.decompose(); + decomposition.write(decomposeSets); + if (writeCellDist) { - // Get corresponding directory name (to handle processors/) - const fileName timePath + writeDecomposition(decomposition); + } + } + + // Clear the field maps if there has been topology change + if + ( + state == fvMesh::TOPO_CHANGE + || state == fvMesh::TOPO_PATCH_CHANGE + ) + { + for (label proci = 0; proci < decomposition.nProcs(); proci++) + { + fieldDecomposerList.set(proci, nullptr); + dimFieldDecomposerList.set(proci, nullptr); + pointFieldDecomposerList.set(proci, nullptr); + } + } + + // Decompose the fields at this time + if (decomposeGeomOnly) + { + // Do nothing + } + else if (copyZero) + { + // Copy the field files from the