diff --git a/applications/utilities/parallelProcessing/decomposePar/decomposePar.C b/applications/utilities/parallelProcessing/decomposePar/decomposePar.C index b06da4f4a7..d1b9b58a7d 100644 --- a/applications/utilities/parallelProcessing/decomposePar/decomposePar.C +++ b/applications/utilities/parallelProcessing/decomposePar/decomposePar.C @@ -330,7 +330,6 @@ int main(int argc, char *argv[]) } - forAll(regionNames, regioni) { const word& regionName = regionNames[regioni]; @@ -338,13 +337,12 @@ int main(int argc, char *argv[]) Info<< "\n\nDecomposing mesh " << regionName << nl << endl; - // Determine the existing processor count directly label nProcs = fileHandler().nProcs(runTime.path(), regionDir); - // Get requested numberOfSubdomains. Note: have no mesh yet so - // cannot use decompositionModel::New - const label nDomains = readLabel + // Get requested numberOfSubdomains directly from the dictionary. + // Note: have no mesh yet so cannot use decompositionModel::New + const label nDomains = decompositionMethod::nDomains ( IOdictionary ( @@ -362,7 +360,7 @@ int main(int argc, char *argv[]) ), decompDictFile ) - ).lookup("numberOfSubdomains") + ) ); if (decomposeFieldsOnly) diff --git a/applications/utilities/parallelProcessing/decomposePar/decomposeParDict b/applications/utilities/parallelProcessing/decomposePar/decomposeParDict index cb061078b7..2ad99e2ea5 100644 --- a/applications/utilities/parallelProcessing/decomposePar/decomposeParDict +++ b/applications/utilities/parallelProcessing/decomposePar/decomposeParDict @@ -15,10 +15,174 @@ FoamFile } // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // -numberOfSubdomains 2; +//- The total number of domains (mandatory) +numberOfSubdomains 256; + +//- The decomposition method (mandatory) +method scotch; +// method hierarchical; +// method simple; +// method metis; +// method manual; +// method multiLevel; +// method structured; // does 2D decomposition of structured mesh -// Optional decomposition constraints +//- Optional region-wise decomposition. +// Can specify a different method. +// The number of subdomains can be less than the top-level numberOfSubdomains. +regions +{ + water + { + numberOfSubdomains 128; + method metis; + } + + ".*solid" + { + numberOfSubdomains 4; + method metis; + } + + heater + { + numberOfSubdomains 1; + method none; + } +} + + +// Coefficients for the decomposition method are either as a +// general "coeffs" dictionary or method-specific "Coeffs". +// For multiLevel, using multiLevelCoeffs only. + + +multiLevelCoeffs +{ + // multiLevel decomposition methods to apply in turn. + // This is like hierarchical but fully general + // - every method can be used at every level. + + // Only sub-dictionaries containing the keyword "method" are used. + // + + level0 + { + numberOfSubdomains 16; + method scotch; + } + level1 + { + numberOfSubdomains 2; + method scotch; + coeffs + { + n (2 1 1); + delta 0.001; + } + } + level2 + { + numberOfSubdomains 8; + // method simple; + method scotch; + } +} + + +multiLevelCoeffs +{ + // Compact multiLevel specification, activated by the presence of the + // keywords "method" and "domains" + + method scotch; + domains (16 2 8); + + //// Or with implicit '16' for the first level with numberOfSubdomains=256 + //domains (2 8); +} + + + +// Other example coefficents + +simpleCoeffs +{ + n (2 1 1); + // delta 0.001; //< default value = 0.001 +} + +hierarchicalCoeffs +{ + n (1 2 1); + // delta 0.001; //< default value = 0.001 + // order xyz; //< default order = xyz +} + +metisCoeffs +{ + /* + processorWeights + ( + 1 + 1 + 1 + 1 + ); + */ +} + +scotchCoeffs +{ + //processorWeights + //( + // 1 + // 1 + // 1 + // 1 + //); + //writeGraph true; + //strategy "b"; +} + +manualCoeffs +{ + dataFile "decompositionData"; +} + +structuredCoeffs +{ + // Patches to do 2D decomposition on. Structured mesh only; cells have + // to be in 'columns' on top of patches. + patches (movingWall); + + // Method to use on the 2D subset + method scotch; +} + + +//- Use the volScalarField named here as a weight for each cell in the +// decomposition. For example, use a particle population field to decompose +// for a balanced number of particles in a lagrangian simulation. +// weightField dsmcRhoNMean; + + +//// Is the case distributed? Note: command-line argument -roots takes +//// precedence +//distributed yes; +// +//// Per slave (so nProcs-1 entries) the directory above the case. +//roots +//( +// "/tmp" +// "/tmp" +//); + + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // +// Decomposition constraints + //constraints //{ // preserveBaffles @@ -88,107 +252,4 @@ numberOfSubdomains 2; //preserveBaffles true; - -//- Use the volScalarField named here as a weight for each cell in the -// decomposition. For example, use a particle population field to decompose -// for a balanced number of particles in a lagrangian simulation. -// weightField dsmcRhoNMean; - -method scotch; -//method hierarchical; -// method simple; -// method metis; -// method manual; -// method multiLevel; -// method structured; // does 2D decomposition of structured mesh - -multiLevelCoeffs -{ - // Decomposition methods to apply in turn. This is like hierarchical but - // fully general - every method can be used at every level. - - level0 - { - numberOfSubdomains 64; - //method simple; - //simpleCoeffs - //{ - // n (2 1 1); - // delta 0.001; - //} - method scotch; - } - level1 - { - numberOfSubdomains 4; - method scotch; - } -} - -// Desired output - -simpleCoeffs -{ - n (2 1 1); - delta 0.001; -} - -hierarchicalCoeffs -{ - n (1 2 1); - delta 0.001; - order xyz; -} - -metisCoeffs -{ - /* - processorWeights - ( - 1 - 1 - 1 - 1 - ); - */ -} - -scotchCoeffs -{ - //processorWeights - //( - // 1 - // 1 - // 1 - // 1 - //); - //writeGraph true; - //strategy "b"; -} - -manualCoeffs -{ - dataFile "decompositionData"; -} - -structuredCoeffs -{ - // Patches to do 2D decomposition on. Structured mesh only; cells have - // to be in 'columns' on top of patches. - patches (movingWall); - - // Method to use on the 2D subset - method scotch; -} - -//// Is the case distributed? Note: command-line argument -roots takes -//// precedence -//distributed yes; -//// Per slave (so nProcs-1 entries) the directory above the case. -//roots -//( -// "/tmp" -// "/tmp" -//); - // ************************************************************************* // diff --git a/applications/utilities/parallelProcessing/decomposePar/domainDecomposition.C b/applications/utilities/parallelProcessing/decomposePar/domainDecomposition.C index bf722cfbae..0cc795761c 100644 --- a/applications/utilities/parallelProcessing/decomposePar/domainDecomposition.C +++ b/applications/utilities/parallelProcessing/decomposePar/domainDecomposition.C @@ -49,10 +49,8 @@ void Foam::domainDecomposition::mark labelList& elementToZone ) { - forAll(zoneElems, i) + for (const label pointi : zoneElems) { - label pointi = zoneElems[i]; - if (elementToZone[pointi] == -1) { // First occurrence @@ -69,7 +67,6 @@ void Foam::domainDecomposition::mark // * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * // -// from components Foam::domainDecomposition::domainDecomposition ( const IOobject& io, @@ -98,13 +95,13 @@ Foam::domainDecomposition::domainDecomposition decompDictFile_(decompDictFile), nProcs_ ( - readInt + decompositionMethod::nDomains ( decompositionModel::New ( *this, decompDictFile - ).lookup("numberOfSubdomains") + ) ) ), distributed_(false), diff --git a/applications/utilities/parallelProcessing/decomposePar/domainDecompositionDistribute.C b/applications/utilities/parallelProcessing/decomposePar/domainDecompositionDistribute.C index 371f44f73e..472077a709 100644 --- a/applications/utilities/parallelProcessing/decomposePar/domainDecompositionDistribute.C +++ b/applications/utilities/parallelProcessing/decomposePar/domainDecompositionDistribute.C @@ -46,11 +46,11 @@ void Foam::domainDecomposition::distributeCells() decompDictFile_ ); + word weightName; scalarField cellWeights; - if (method.found("weightField")) - { - word weightName = method.lookup("weightField"); + if (method.readIfPresent("weightField", weightName)) + { volScalarField weights ( IOobject diff --git a/applications/utilities/preProcessing/mapFields/mapFields.C b/applications/utilities/preProcessing/mapFields/mapFields.C index 93ba316412..be6261d3f0 100644 --- a/applications/utilities/preProcessing/mapFields/mapFields.C +++ b/applications/utilities/preProcessing/mapFields/mapFields.C @@ -58,7 +58,7 @@ int readNumProcs dictFile = dictFile / dictName; } - return readInt + return decompositionMethod::nDomains ( IOdictionary ( @@ -75,7 +75,7 @@ int readNumProcs ), dictFile ) - ).lookup("numberOfSubdomains") + ) ); } diff --git a/src/dummyThirdParty/kahipDecomp/dummyKahipDecomp.C b/src/dummyThirdParty/kahipDecomp/dummyKahipDecomp.C index 96e0bdc16b..6e147e2626 100644 --- a/src/dummyThirdParty/kahipDecomp/dummyKahipDecomp.C +++ b/src/dummyThirdParty/kahipDecomp/dummyKahipDecomp.C @@ -73,10 +73,20 @@ Foam::label Foam::kahipDecomp::decomposeSerial Foam::kahipDecomp::kahipDecomp ( - const dictionary& decompositionDict + const dictionary& decompDict ) : - metisLikeDecomp(decompositionDict) + metisLikeDecomp("kahip", decompDict) +{} + + +Foam::kahipDecomp::kahipDecomp +( + const dictionary& decompDict, + const word& regionName +) +: + metisLikeDecomp("kahip", decompDict, regionName) {} diff --git a/src/dummyThirdParty/metisDecomp/dummyMetisDecomp.C b/src/dummyThirdParty/metisDecomp/dummyMetisDecomp.C index 3743cc1bae..c5f8e3ba9a 100644 --- a/src/dummyThirdParty/metisDecomp/dummyMetisDecomp.C +++ b/src/dummyThirdParty/metisDecomp/dummyMetisDecomp.C @@ -73,10 +73,20 @@ Foam::label Foam::metisDecomp::decomposeSerial Foam::metisDecomp::metisDecomp ( - const dictionary& decompositionDict + const dictionary& decompDict ) : - metisLikeDecomp(decompositionDict) + metisLikeDecomp("metis", decompDict) +{} + + +Foam::metisDecomp::metisDecomp +( + const dictionary& decompDict, + const word& regionName +) +: + metisLikeDecomp("metis", decompDict, regionName) {} diff --git a/src/dummyThirdParty/ptscotchDecomp/dummyPtscotchDecomp.C b/src/dummyThirdParty/ptscotchDecomp/dummyPtscotchDecomp.C index decacb6c10..5a756b477f 100644 --- a/src/dummyThirdParty/ptscotchDecomp/dummyPtscotchDecomp.C +++ b/src/dummyThirdParty/ptscotchDecomp/dummyPtscotchDecomp.C @@ -98,10 +98,22 @@ Foam::label Foam::ptscotchDecomp::decompose Foam::ptscotchDecomp::ptscotchDecomp ( - const dictionary& decompositionDict + const dictionary& decompDict ) : - decompositionMethod(decompositionDict) + decompositionMethod(decompDict), + coeffsDict_(dictionary::null) +{} + + +Foam::ptscotchDecomp::ptscotchDecomp +( + const dictionary& decompDict, + const word& regionName +) +: + decompositionMethod(decompDict, regionName), + coeffsDict_(dictionary::null) {} diff --git a/src/dummyThirdParty/scotchDecomp/dummyScotchDecomp.C b/src/dummyThirdParty/scotchDecomp/dummyScotchDecomp.C index 582c770253..9aa8e20ec0 100644 --- a/src/dummyThirdParty/scotchDecomp/dummyScotchDecomp.C +++ b/src/dummyThirdParty/scotchDecomp/dummyScotchDecomp.C @@ -80,10 +80,20 @@ Foam::label Foam::scotchDecomp::decomposeSerial Foam::scotchDecomp::scotchDecomp ( - const dictionary& decompositionDict + const dictionary& decompDict ) : - metisLikeDecomp(decompositionDict) + metisLikeDecomp("scotch", decompDict) +{} + + +Foam::scotchDecomp::scotchDecomp +( + const dictionary& decompDict, + const word& regionName +) +: + metisLikeDecomp("scotch", decompDict, regionName) {} diff --git a/src/parallel/decompose/decompose/decompositionModel.C b/src/parallel/decompose/decompose/decompositionModel.C index dbcb023af1..8f156c64b5 100644 --- a/src/parallel/decompose/decompose/decompositionModel.C +++ b/src/parallel/decompose/decompose/decompositionModel.C @@ -34,6 +34,7 @@ namespace Foam defineTypeNameAndDebug(decompositionModel, 0); } + // * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * // Foam::decompositionModel::decompositionModel diff --git a/src/parallel/decompose/decompose/decompositionModel.H b/src/parallel/decompose/decompose/decompositionModel.H index 7ed6bd3fa2..2cbfa4d249 100644 --- a/src/parallel/decompose/decompose/decompositionModel.H +++ b/src/parallel/decompose/decompose/decompositionModel.H @@ -75,14 +75,14 @@ public: // Selectors - //- Read (optionallly from absolute path) & register on mesh + //- Read (optionally from absolute path) & register on mesh static const decompositionModel& New ( const polyMesh& mesh, const fileName& decompDictFile = "" ); - //- Read (optionallly from supplied dictionary) & register on mesh + //- Read (optionally from supplied dictionary) & register on mesh static const decompositionModel& New ( const polyMesh& mesh, @@ -94,42 +94,51 @@ public: // Constructors //- Construct from typeName or optional path to controlDictionary - decompositionModel(const polyMesh&, const fileName& = ""); + decompositionModel + ( + const polyMesh& mesh, + const fileName& decompDictFile = "" + ); //- Construct from typeName or optional path to controlDictionary decompositionModel ( - const polyMesh&, + const polyMesh& mesh, const dictionary& dict, - const fileName& = "" + const fileName& decompDictFile = "" ); - // Member functions + // Member Functions decompositionMethod& decomposer() const { if (!decomposerPtr_.valid()) { - decomposerPtr_ = decompositionMethod::New(*this); + decomposerPtr_ = + decompositionMethod::New + ( + *this, + this->mesh().name() // Name of mesh region + ); } return decomposerPtr_(); } //- Helper: return IOobject with optionally absolute path provided - static IOobject selectIO(const IOobject&, const fileName&); + static IOobject selectIO(const IOobject& io, const fileName& f); - // UpdateableMeshObject + // UpdateableMeshObject - virtual bool movePoints() - { - return false; - } + virtual bool movePoints() + { + return false; + } - virtual void updateMesh(const mapPolyMesh&) - {} + virtual void updateMesh(const mapPolyMesh&) + {} }; diff --git a/src/parallel/decompose/decomposition.dox b/src/parallel/decompose/decomposition.dox new file mode 100644 index 0000000000..0256b48e4a --- /dev/null +++ b/src/parallel/decompose/decomposition.dox @@ -0,0 +1,282 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | OpenFOAM: The Open Source CFD Toolbox + \\ / O peration | + \\ / A nd | Copyright (C) 2017 OpenCFD Ltd. + \\/ M anipulation | +------------------------------------------------------------------------------- +License + This file is part of OpenFOAM. + + OpenFOAM is free software: you can redistribute it and/or modify it under + the terms of the GNU General Public License as published by the Free + Software Foundation, either version 3 of the License, or (at your option) + any later version. + + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + details. + + You should have received a copy of the GNU General Public License along with + OpenFOAM. If not, see . + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +\page domainDecomposition Domain decomposition + +\section secIntroduction Introduction + +When running a simulation in parallel, the geometry must first be decomposed +(segmented) into individual geometries for each MPI process. These separate +geometries are \a connected together with special processor boundary patches. +The processor-specific \c constant/polyMesh/boundary files will contain this +type of entry: + +\verbatim + procBoundary0to14 + { + type processor; + inGroups 1(processor); + nFaces 131; + startFace 34983; + matchTolerance 0.0001; + transform unknown; + myProcNo 0; + neighbProcNo 14; + } +\endverbatim + +The decomposePar utility is a commonly used method to decompose domains +and subsequently distribute the fields. +The reconstructPar and reconstructParMesh utilities +can be used to reconstruct a single domain from the processor sub-domains. + +\section secDecomposeParDict The decomposeParDict + +The \c decomposeParDict is required by decompose utilities and for any solvers +or utilities running in parallel. It is normally located in the simulation +\c system directory. The -decomposeParDict name command-line option +can be used to specify an alternate file. + +The \a numberOfSubdomains entry is mandatory: +\verbatim + numberOfSubdomains ; +\endverbatim + +The \a method entry is required for the decomposePar utility and specifies +the decomposition method type: +\verbatim + method ; +\endverbatim +The \a method entry is generally not required when running a simulation. + +OpenFOAM offers a variety of decomposition methods and interfaces to external, +third-party decomposition routines. The types of decomposition methods +available will thus depend on your particular installation. + +\table + Name | Class + none | #Foam::noDecomp + manual | #Foam::manualDecomp + simple | #Foam::simpleGeomDecomp + hierarchical | #Foam::hierarchGeomDecomp + kahip | #Foam::kahipDecomp + metis | #Foam::metisDecomp + scotch | #Foam::scotchDecomp + structured | #Foam::structuredDecomp + multiLevel | #Foam::multiLevelDecomp +\endtable + +If a decomposition method requires any additional configuration controls, +these are specified either within in a generic \c coeffs dictionary that +or a method-specific version. For example, + +\verbatim + method hierarchical; + + coeffs + { + n (4 2 3); + } + + // ----- + + method metis; + + metisCoeffs + { + method k-way; + } +\endverbatim + +For simplicity, the generic \c coeffs dictionary is generally preferrable. +However, for some specific decomposition methods +(eg, \ref subsecMultiLevel "multiLevel") only the +method-specific coefficients dictionary is permitted. + + +\subsection subsecRegions Multi-region + +When running multi-region simulations, it may be desirable to use different +decomposition methods for one or more regions, or even to have fewer +processors allocated to a particular region. +If, for example, the multi-region simulation contains a large fluid region +and a very small solid region, it can be advantageous to decompose the solid +onto fewer processors. + +The region-wise specification is contained in a \a regions subdictionary with +decomposeParDict. For example, + +\verbatim + numberOfSubdomains 2048; + method metis; + + regions + { + heater + { + numberOfSubdomains 2; + method hierarchical; + coeffs + { + n (2 1 1); + } + } + + "*.solid" + { + numberOfSubdomains 16; + method scotch; + } + } +\endverbatim + +\note +The top-level numberOfSubdomains remains mandatory, since this specifies the +number of domains for the entire simulation. The individual regions may use +the same number or fewer domains. The \a numberOfSubdomains entry within +a region specification is only needed if the value differs. + + +\subsection subsecMultiLevel Multi-level decomposition + +The #Foam::multiLevelDecomp decomposition provides a general means of +successively decomposing with different methods. Each appplication of the +decomposition is termed a level. For example, + +\verbatim + numberOfSubdomains 2048; + method multiLevel; + + multiLevelCoeffs + { + nodes + { + numberOfSubdomains 128; + method hierarchical; + coeffs + { + n (16 4 2); + } + } + cpus + { + numberOfSubdomains 2; + method scotch; + } + cores + { + numberOfSubdomains 8; + method metis; + } + } +\endverbatim + +For cases where the same method is applied at each level, this can also be +conveniently written in a much shorter form: +\verbatim + numberOfSubdomains 2048; + method multiLevel; + + multiLevelCoeffs + { + method scotch + domains (128 2 8); + } +\endverbatim + +When the specified \a domains is smaller than \a numberOfSubdomains +but can be resolved as an integral multiple, this integral multiple +is used as the first level. This can make it easier to manage when +changing the number of domains for the simulation. +For example, +\verbatim + numberOfSubdomains 1024; + method multiLevel; + + multiLevelCoeffs + { + method scotch + domains (2 8); //< inferred as domains (64 2 8); + } +\endverbatim + + +\subsection subsecConstraints Constraints + +\verbatim + constraints + { + preserveBaffles + { + // Keep owner and neighbour of baffles on same processor + // (ie, keep it detectable as a baffle). + // Baffles are two boundary face sharing the same points + + type preserveBaffles; + } + + preserveFaceZones + { + // Keep owner and neighbour on same processor for faces in zones + + type preserveFaceZones; + zones (".*"); + } + + preservePatches + { + // Keep owner and neighbour on same processor for faces in patches + // (only makes sense for cyclic patches. Not suitable for e.g. + // cyclicAMI since these are not coupled on the patch level. + // Use singleProcessorFaceSets for those. + + type preservePatches; + patches (".*"); + } + + singleProcessorFaceSets + { + // Keep all of faceSet on a single processor. This puts all cells + // connected with a point, edge or face on the same processor. + // (just having face connected cells might not guarantee a balanced + // decomposition) + // The processor can be -1 (the decompositionMethod chooses the + // processor for a good load balance) or explicitly provided (upsets + // balance) + + type singleProcessorFaceSets; + singleProcessorFaceSets ((f1 -1)); + } + + refinementHistory + { + // Decompose cells such that all cell originating from single cell + // end up on same processor + type refinementHistory; + } + } +\endverbatim + +\*---------------------------------------------------------------------------*/ diff --git a/src/parallel/decompose/decompositionMethods/decompositionMethod/decompositionMethod.C b/src/parallel/decompose/decompositionMethods/decompositionMethod/decompositionMethod.C index c17958367a..69bc26b4d3 100644 --- a/src/parallel/decompose/decompositionMethods/decompositionMethod/decompositionMethod.C +++ b/src/parallel/decompose/decompositionMethods/decompositionMethod/decompositionMethod.C @@ -44,49 +44,97 @@ namespace Foam { defineTypeNameAndDebug(decompositionMethod, 0); defineRunTimeSelectionTable(decompositionMethod, dictionary); + defineRunTimeSelectionTable(decompositionMethod, dictionaryRegion); + + // Fallback name when searching for optional coefficients directories + static const word defaultName("coeffs"); } -// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * // +// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * // -Foam::decompositionMethod::decompositionMethod -( - const dictionary& decompositionDict -) -: - decompositionDict_(decompositionDict), - nProcessors_ - ( - readLabel(decompositionDict.lookup("numberOfSubdomains")) - ) +Foam::label Foam::decompositionMethod::nDomains(const dictionary& decompDict) { - // Read any constraints - wordList constraintTypes_; - if (decompositionDict_.found("constraints")) + return readLabel(decompDict.lookup("numberOfSubdomains")); +} + + +Foam::label Foam::decompositionMethod::nDomains +( + const dictionary& decompDict, + const word& regionName +) +{ + const label nDomainsGlobal = nDomains(decompDict); + + const dictionary& regionDict(optionalRegionDict(decompDict, regionName)); + + label nDomainsRegion; + if (regionDict.readIfPresent("numberOfSubdomains", nDomainsRegion)) { - //PtrList constraintsList - //( - // decompositionDict_.lookup("constraints") - //); - //forAll(constraintsList, i) - //{ - // const dictionary& dict = constraintsList[i]; - const dictionary& constraintsList = decompositionDict_.subDict - ( - "constraints" - ); - forAllConstIter(dictionary, constraintsList, iter) + if (nDomainsRegion >= 1 && nDomainsRegion <= nDomainsGlobal) + { + return nDomainsRegion; + } + + WarningInFunction + << "ignoring out of range numberOfSubdomains " + << nDomainsRegion << " for region " << regionName + << nl << nl + << endl; + } + + return nDomainsGlobal; +} + + +const Foam::dictionary& Foam::decompositionMethod::optionalRegionDict +( + const dictionary& decompDict, + const word& regionName +) +{ + auto finder = decompDict.csearch("regions"); + + if (!regionName.empty() && finder.isDict()) + { + finder = finder.dict().csearch(regionName); + + if (finder.isDict()) + { + return finder.dict(); + } + } + + return dictionary::null; +} + + +// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * // + +void Foam::decompositionMethod::readConstraints() +{ + constraints_.clear(); + + // Read any constraints + wordList constraintTypes; + + const dictionary* dictptr = decompositionDict_.subDictPtr("constraints"); + + if (dictptr) + { + forAllConstIters(*dictptr, iter) { const dictionary& dict = iter().dict(); - constraintTypes_.append(dict.lookup("type")); + constraintTypes.append(dict.lookup("type")); constraints_.append ( decompositionConstraint::New ( dict, - constraintTypes_.last() + constraintTypes.last() ) ); } @@ -96,7 +144,7 @@ Foam::decompositionMethod::decompositionMethod if ( decompositionDict_.found("preserveBaffles") - && !constraintTypes_.found + && !constraintTypes.found ( decompositionConstraints::preserveBafflesConstraint::typeName ) @@ -111,7 +159,7 @@ Foam::decompositionMethod::decompositionMethod if ( decompositionDict_.found("preservePatches") - && !constraintTypes_.found + && !constraintTypes.found ( decompositionConstraints::preservePatchesConstraint::typeName ) @@ -128,7 +176,7 @@ Foam::decompositionMethod::decompositionMethod if ( decompositionDict_.found("preserveFaceZones") - && !constraintTypes_.found + && !constraintTypes.found ( decompositionConstraints::preserveFaceZonesConstraint::typeName ) @@ -145,7 +193,7 @@ Foam::decompositionMethod::decompositionMethod if ( decompositionDict_.found("singleProcessorFaceSets") - && !constraintTypes_.found + && !constraintTypes.found ( decompositionConstraints::preserveFaceZonesConstraint::typeName ) @@ -166,17 +214,144 @@ Foam::decompositionMethod::decompositionMethod } } +// * * * * * * * * * * * * Protected Member Functions * * * * * * * * * * * // + +const Foam::dictionary& Foam::decompositionMethod::findCoeffsDict +( + const dictionary& dict, + const word& coeffsName, + int select +) +{ + dictionary::const_searcher fnd; + + if + ( + (fnd = dict.csearch(coeffsName)).isDict() + || + ( + !(select & selectionType::EXACT) + && (fnd = dict.csearch(defaultName)).isDict() + ) + ) + { + return fnd.dict(); + } + + // Not found + if (select & selectionType::MANDATORY) + { + FatalIOError + << "'" << coeffsName << "' dictionary not found in dictionary " + << dict.name() << endl + << abort(FatalIOError); + } + + if (select & selectionType::NULL_DICT) + { + return dictionary::null; + } + + return dict; +} + + +const Foam::dictionary& Foam::decompositionMethod::findCoeffsDict +( + const word& coeffsName, + int select +) const +{ + dictionary::const_searcher fnd; + + if + ( + !decompositionRegionDict_.empty() + && + ( + (fnd = decompositionRegionDict_.csearch(coeffsName)).isDict() + || + ( + !(select & selectionType::EXACT) + && (fnd = decompositionRegionDict_.csearch(defaultName)).isDict() + ) + ) + ) + { + return fnd.dict(); + } + + if + ( + (fnd = decompositionDict_.csearch(coeffsName)).isDict() + || + ( + !(select & selectionType::EXACT) + && (fnd = decompositionDict_.csearch(defaultName)).isDict() + ) + ) + { + return fnd.dict(); + } + + // Not found + if (select & selectionType::MANDATORY) + { + FatalIOError + << "'" << coeffsName << "' dictionary not found in dictionary " + << decompositionDict_.name() << endl + << abort(FatalIOError); + } + + if (select & selectionType::NULL_DICT) + { + return dictionary::null; + } + + return decompositionDict_; +} + + +// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * // + +Foam::decompositionMethod::decompositionMethod +( + const dictionary& decompDict +) +: + decompositionDict_(decompDict), + decompositionRegionDict_(dictionary::null), + nDomains_(nDomains(decompDict)) +{ + readConstraints(); +} + + +Foam::decompositionMethod::decompositionMethod +( + const dictionary& decompDict, + const word& regionName +) +: + decompositionDict_(decompDict), + decompositionRegionDict_ + ( + optionalRegionDict(decompositionDict_, regionName) + ), + nDomains_(nDomains(decompDict, regionName)) +{ + readConstraints(); +} + // * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // Foam::autoPtr Foam::decompositionMethod::New ( - const dictionary& decompositionDict + const dictionary& decompDict ) { - const word methodType(decompositionDict.lookup("method")); - - Info<< "Selecting decompositionMethod " << methodType << endl; + const word methodType(decompDict.lookup("method")); auto cstrIter = dictionaryConstructorTablePtr_->cfind(methodType); @@ -190,7 +365,58 @@ Foam::autoPtr Foam::decompositionMethod::New << exit(FatalError); } - return autoPtr(cstrIter()(decompositionDict)); + // verbose + { + Info<< "Selecting decompositionMethod " << methodType + << " [" << (nDomains(decompDict)) << "]" << endl; + } + + return autoPtr(cstrIter()(decompDict)); +} + + +Foam::autoPtr Foam::decompositionMethod::New +( + const dictionary& decompDict, + const word& regionName + +) +{ + const dictionary& regionDict(optionalRegionDict(decompDict, regionName)); + + if (regionDict.empty()) + { + // No region-specific information - just forward to normal routine + return decompositionMethod::New(decompDict); + } + + word methodType(decompDict.lookup("method")); + regionDict.readIfPresent("method", methodType); + + auto cstrIter = dictionaryRegionConstructorTablePtr_->cfind(methodType); + + if (!cstrIter.found()) + { + WarningInFunction + << nl + << "Unknown region decompositionMethod " + << methodType << nl << nl + << "Valid decompositionMethods : " << endl + << dictionaryRegionConstructorTablePtr_->sortedToc() << nl + << "Reverting to non-region version" << nl + << endl; + + return decompositionMethod::New(decompDict); + } + + // verbose + { + Info<< "Selecting decompositionMethod " << methodType + << " [" << (nDomains(decompDict, regionName)) << "] (region " + << regionName << ")" << endl; + } + + return autoPtr(cstrIter()(decompDict, regionName)); } @@ -341,7 +567,7 @@ void Foam::decompositionMethod::calcCellCells // Number of faces per coarse cell labelList nFacesPerCell(nLocalCoarse, 0); - for (label facei = 0; facei < mesh.nInternalFaces(); facei++) + for (label facei = 0; facei < mesh.nInternalFaces(); ++facei) { const label own = agglom[faceOwner[facei]]; const label nei = agglom[faceNeighbour[facei]]; @@ -391,7 +617,7 @@ void Foam::decompositionMethod::calcCellCells const labelList& offsets = cellCells.offsets(); // For internal faces is just offsetted owner and neighbour - for (label facei = 0; facei < mesh.nInternalFaces(); facei++) + for (label facei = 0; facei < mesh.nInternalFaces(); ++facei) { const label own = agglom[faceOwner[facei]]; const label nei = agglom[faceNeighbour[facei]]; @@ -453,7 +679,7 @@ void Foam::decompositionMethod::calcCellCells const label endIndex = cellCells.offsets()[celli+1]; - for (label i = startIndex; i < endIndex; i++) + for (label i = startIndex; i < endIndex; ++i) { if (nbrCells.insert(cellCells.m()[i])) { @@ -547,10 +773,10 @@ void Foam::decompositionMethod::calcCellCells // Number of faces per coarse cell labelList nFacesPerCell(nLocalCoarse, 0); - for (label faceI = 0; faceI < mesh.nInternalFaces(); faceI++) + for (label facei = 0; facei < mesh.nInternalFaces(); ++facei) { - const label own = agglom[faceOwner[faceI]]; - const label nei = agglom[faceNeighbour[faceI]]; + const label own = agglom[faceOwner[facei]]; + const label nei = agglom[faceNeighbour[facei]]; nFacesPerCell[own]++; nFacesPerCell[nei]++; @@ -599,18 +825,18 @@ void Foam::decompositionMethod::calcCellCells const labelList& offsets = cellCells.offsets(); // For internal faces is just offsetted owner and neighbour - for (label faceI = 0; faceI < mesh.nInternalFaces(); faceI++) + for (label facei = 0; facei < mesh.nInternalFaces(); ++facei) { - const label own = agglom[faceOwner[faceI]]; - const label nei = agglom[faceNeighbour[faceI]]; + const label own = agglom[faceOwner[facei]]; + const label nei = agglom[faceNeighbour[facei]]; const label ownIndex = offsets[own] + nFacesPerCell[own]++; const label neiIndex = offsets[nei] + nFacesPerCell[nei]++; m[ownIndex] = globalAgglom.toGlobal(nei); - w[ownIndex] = mag(mesh.faceAreas()[faceI]); + w[ownIndex] = mag(mesh.faceAreas()[facei]); m[neiIndex] = globalAgglom.toGlobal(own); - w[ownIndex] = mag(mesh.faceAreas()[faceI]); + w[ownIndex] = mag(mesh.faceAreas()[facei]); } // For boundary faces is offsetted coupled neighbour @@ -661,14 +887,14 @@ void Foam::decompositionMethod::calcCellCells label startIndex = cellCells.offsets()[0]; - forAll(cellCells, cellI) + forAll(cellCells, celli) { nbrCells.clear(); - nbrCells.insert(globalAgglom.toGlobal(cellI)); + nbrCells.insert(globalAgglom.toGlobal(celli)); - const label endIndex = cellCells.offsets()[cellI+1]; + const label endIndex = cellCells.offsets()[celli+1]; - for (label i = startIndex; i < endIndex; i++) + for (label i = startIndex; i < endIndex; ++i) { if (nbrCells.insert(cellCells.m()[i])) { @@ -678,8 +904,8 @@ void Foam::decompositionMethod::calcCellCells } } startIndex = endIndex; - cellCells.offsets()[cellI+1] = newIndex; - cellCellWeights.offsets()[cellI+1] = newIndex; + cellCells.offsets()[celli+1] = newIndex; + cellCellWeights.offsets()[celli+1] = newIndex; } cellCells.m().setSize(newIndex); @@ -753,7 +979,7 @@ void Foam::decompositionMethod::calcCellCells // labelList nFacesPerCell(nLocalCoarse, 0); // // // 1. Internal faces -// for (label facei = 0; facei < mesh.nInternalFaces(); facei++) +// for (label facei = 0; facei < mesh.nInternalFaces(); ++facei) // { // if (!blockedFace[facei]) // { @@ -880,7 +1106,7 @@ void Foam::decompositionMethod::calcCellCells // const labelList& offsets = cellCells.offsets(); // // // 1. For internal faces is just offsetted owner and neighbour -// for (label facei = 0; facei < mesh.nInternalFaces(); facei++) +// for (label facei = 0; facei < mesh.nInternalFaces(); ++facei) // { // if (!blockedFace[facei]) // { @@ -1032,7 +1258,7 @@ void Foam::decompositionMethod::calcCellCells // // label endIndex = cellCells.offsets()[celli+1]; // -// for (label i = startIndex; i < endIndex; i++) +// for (label i = startIndex; i < endIndex; ++i) // { // if (nbrCells.insert(cellCells.m()[i])) // { @@ -1180,11 +1406,11 @@ Foam::labelList Foam::decompositionMethod::decompose forAll(localRegion, celli) { - label regionI = localRegion[celli]; + const label regioni = localRegion[celli]; - if (regionCentres[regionI] == point::max) + if (regionCentres[regioni] == point::max) { - regionCentres[regionI] = mesh.cellCentres()[celli]; + regionCentres[regioni] = mesh.cellCentres()[celli]; } } @@ -1197,18 +1423,18 @@ Foam::labelList Foam::decompositionMethod::decompose { forAll(localRegion, celli) { - label regionI = localRegion[celli]; + const label regioni = localRegion[celli]; - regionWeights[regionI] += cellWeights[celli]; + regionWeights[regioni] += cellWeights[celli]; } } else { forAll(localRegion, celli) { - label regionI = localRegion[celli]; + const label regioni = localRegion[celli]; - regionWeights[regionI] += 1.0; + regionWeights[regioni] += 1.0; } } @@ -1224,11 +1450,11 @@ Foam::labelList Foam::decompositionMethod::decompose // Implement the explicitConnections since above decompose // does not know about them - forAll(explicitConnections, i) + forAll(explicitConnections, connectioni) { - const labelPair& baffle = explicitConnections[i]; - label f0 = baffle.first(); - label f1 = baffle.second(); + const labelPair& baffle = explicitConnections[connectioni]; + const label f0 = baffle.first(); + const label f1 = baffle.second(); if (!blockedFace[f0] && !blockedFace[f1]) { @@ -1292,7 +1518,7 @@ Foam::labelList Foam::decompositionMethod::decompose { if (!blockedFace[facei]) { - label own = mesh.faceOwner()[facei]; + const label own = mesh.faceOwner()[facei]; seedFaces[nUnblocked] = facei; seedData[nUnblocked] = minData(finalDecomp[own]); nUnblocked++; @@ -1361,10 +1587,8 @@ Foam::labelList Foam::decompositionMethod::decompose forAll(f, fp) { const labelList& pFaces = mesh.pointFaces()[f[fp]]; - forAll(pFaces, i) + for (const label facei : pFaces) { - label facei = pFaces[i]; - finalDecomp[mesh.faceOwner()[facei]] = proci; if (mesh.isInternalFace(facei)) { @@ -1433,9 +1657,9 @@ void Foam::decompositionMethod::setConstraints specifiedProcessorFaces.clear(); explicitConnections.clear(); - forAll(constraints_, constraintI) + forAll(constraints_, constrainti) { - constraints_[constraintI].add + constraints_[constrainti].add ( mesh, blockedFace, @@ -1457,9 +1681,9 @@ void Foam::decompositionMethod::applyConstraints labelList& decomposition ) { - forAll(constraints_, constraintI) + forAll(constraints_, constrainti) { - constraints_[constraintI].apply + constraints_[constrainti].apply ( mesh, blockedFace, diff --git a/src/parallel/decompose/decompositionMethods/decompositionMethod/decompositionMethod.H b/src/parallel/decompose/decompositionMethods/decompositionMethod/decompositionMethod.H index 75a797e1ae..96ad3afc98 100644 --- a/src/parallel/decompose/decompositionMethods/decompositionMethod/decompositionMethod.H +++ b/src/parallel/decompose/decompositionMethods/decompositionMethod/decompositionMethod.H @@ -25,7 +25,7 @@ Class Foam::decompositionMethod Description - Abstract base class for decomposition + Abstract base class for domain decomposition SourceFiles decompositionMethod.C @@ -43,29 +43,86 @@ namespace Foam { /*---------------------------------------------------------------------------*\ - Class decompositionMethod Declaration + Class decompositionMethod Declaration \*---------------------------------------------------------------------------*/ class decompositionMethod { + // Private Member Functions + + //- Set PtrList of constraints by reading decompositionDict_. + void readConstraints(); + + //- Disallow default bitwise copy construct and assignment + decompositionMethod(const decompositionMethod&) = delete; + void operator=(const decompositionMethod&) = delete; + protected: + //- Selection type when handling the coefficients dictionary. + // To be used as a bit-mask for findCoeffsDict + enum selectionType + { + DEFAULT = 0, //!< Default request + EXACT = 1, //!< No fallback to "coeffs" if main name not found + MANDATORY = 2, //!< Fatal if dictionary could not be found + NULL_DICT = 4, //!< On failure, return dictionary::null instead + //!< of the top-level enclosing dictionary. + }; + + // Protected data + //- Top-level decomposition dictionary (eg, decomposeParDict) const dictionary& decompositionDict_; - label nProcessors_; + + //- Region-specific decomposition dictionary information + const dictionary& decompositionRegionDict_; + + //- Number of domains for the decomposition + label nDomains_; //- Optional constraints PtrList constraints_; -private: - // Private Member Functions + // Protected Member Functions - //- Disallow default bitwise copy construct and assignment - decompositionMethod(const decompositionMethod&); - void operator=(const decompositionMethod&); + //- Locate coeffsName dictionary or the fallback "coeffs" dictionary + //- within an enclosing dictionary. + // + // \param select choose to include "coeffs" in the search, make + // failure a FatalError, return dictionary::null instead on + // failure. + // + // \return the coefficients dictionary found. If nothing was found, + // return the enclosing dictionary or + // dictionary::null (depending on the select parameter). + static const dictionary& findCoeffsDict + ( + const dictionary& dict, + const word& coeffsName, + int select = selectionType::DEFAULT + ); + + + //- Locate coeffsName dictionary or the fallback "coeffs" dictionary. + // Searches both the region-specific decomposition dictionary + // and the top-level decomposition dictionary. + // + // \param select choose to include "coeffs" in the search, make + // failure a FatalError, return dictionary::null instead on + // failure. + // + // \return the coefficients dictionary found. If nothing was found, + // return the top-level (non-region) dictionary or + // dictionary::null (depending on the select parameter). + const dictionary& findCoeffsDict + ( + const word& coeffsName, + int select = selectionType::DEFAULT + ) const; public: @@ -82,9 +139,43 @@ public: decompositionMethod, dictionary, ( - const dictionary& decompositionDict + const dictionary& decompDict ), - (decompositionDict) + (decompDict) + ); + + declareRunTimeSelectionTable + ( + autoPtr, + decompositionMethod, + dictionaryRegion, + ( + const dictionary& decompDict, + const word& regionName + ), + (decompDict, regionName) + ); + + + // Static Methods + + //- Return the \c numberOfSubdomains entry from the dictionary + static label nDomains(const dictionary& decompDict); + + //- Return the \c numberOfSubdomains from a region within the + // "regions" sub-dictionary + static label nDomains + ( + const dictionary& decompDict, + const word& regionName + ); + + //- Return an optional region dictionary from "regions" sub-dictionary + // or dictionary::null on failure. + static const dictionary& optionalRegionDict + ( + const dictionary& decompDict, + const word& regionName ); @@ -93,14 +184,29 @@ public: //- Return a reference to the selected decomposition method static autoPtr New ( - const dictionary& decompositionDict + const dictionary& decompDict + ); + + //- Return a reference to the selected decomposition method, + //- with region specification + static autoPtr New + ( + const dictionary& decompDict, + const word& regionName ); // Constructors //- Construct given the decomposition dictionary - decompositionMethod(const dictionary& decompositionDict); + decompositionMethod(const dictionary& decompDict); + + //- Construct given the decomposition dictionary for specific region + decompositionMethod + ( + const dictionary& decompDict, + const word& regionName + ); //- Destructor @@ -110,9 +216,10 @@ public: // Member Functions - label nDomains() const + //- Number of domains + inline label nDomains() const { - return nProcessors_; + return nDomains_; } //- Is method parallel aware (i.e. does it synchronize domains across diff --git a/src/parallel/decompose/decompositionMethods/geomDecomp/geomDecomp.C b/src/parallel/decompose/decompositionMethods/geomDecomp/geomDecomp.C index 6f741b0959..1c2df26498 100644 --- a/src/parallel/decompose/decompositionMethods/geomDecomp/geomDecomp.C +++ b/src/parallel/decompose/decompositionMethods/geomDecomp/geomDecomp.C @@ -25,36 +25,29 @@ License #include "geomDecomp.H" -// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * // +// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * // -Foam::geomDecomp::geomDecomp -( - const dictionary& decompositionDict, - const word& derivedType -) -: - decompositionMethod(decompositionDict), - geomDecomDict_(decompositionDict.optionalSubDict(derivedType + "Coeffs")), - n_(geomDecomDict_.lookup("n")), - delta_(readScalar(geomDecomDict_.lookup("delta"))), - rotDelta_(I) +void Foam::geomDecomp::readCoeffs() { - // check that the case makes sense : + coeffsDict_.readIfPresent("delta", delta_); - if (nProcessors_ != n_.x()*n_.y()*n_.z()) + coeffsDict_.lookup("n") >> n_; + + // Verify that the input makes sense + if (nDomains_ != n_.x()*n_.y()*n_.z()) { FatalErrorInFunction - << "Wrong number of processor divisions in geomDecomp:" << nl - << "Number of domains : " << nProcessors_ << nl + << "Wrong number of domain divisions in geomDecomp:" << nl + << "Number of domains : " << nDomains_ << nl << "Wanted decomposition : " << n_ << exit(FatalError); } - scalar d = 1 - 0.5*delta_*delta_; - scalar d2 = sqr(d); + const scalar d = 1 - 0.5*delta_*delta_; + const scalar d2 = sqr(d); - scalar a = delta_; - scalar a2 = sqr(a); + const scalar a = delta_; + const scalar a2 = sqr(a); rotDelta_ = tensor ( @@ -65,4 +58,41 @@ Foam::geomDecomp::geomDecomp } +// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * // + +Foam::geomDecomp::geomDecomp +( + const word& derivedType, + const dictionary& decompDict, + int select +) +: + decompositionMethod(decompDict), + coeffsDict_(findCoeffsDict(derivedType + "Coeffs", select)), + n_(1,1,1), + delta_(0.001), + rotDelta_(I) +{ + readCoeffs(); +} + + +Foam::geomDecomp::geomDecomp +( + const word& derivedType, + const dictionary& decompDict, + const word& regionName, + int select +) +: + decompositionMethod(decompDict, regionName), + coeffsDict_(findCoeffsDict(derivedType + "Coeffs", select)), + n_(1,1,1), + delta_(0.001), + rotDelta_(I) +{ + readCoeffs(); +} + + // ************************************************************************* // diff --git a/src/parallel/decompose/decompositionMethods/geomDecomp/geomDecomp.H b/src/parallel/decompose/decompositionMethods/geomDecomp/geomDecomp.H index e1538356ee..7684a2ef4e 100644 --- a/src/parallel/decompose/decompositionMethods/geomDecomp/geomDecomp.H +++ b/src/parallel/decompose/decompositionMethods/geomDecomp/geomDecomp.H @@ -25,7 +25,14 @@ Class Foam::geomDecomp Description - Geometrical domain decomposition + Base for geometrical domain decomposition methods + + Base coefficients: + \table + Property | Description | Required | Default + n | (nx ny nz) | yes + delta | delta for rotation matrix | no | 0.001 + \endtable SourceFiles geomDecomp.C @@ -42,36 +49,56 @@ namespace Foam { /*---------------------------------------------------------------------------*\ - Class geomDecomp Declaration + Class geomDecomp Declaration \*---------------------------------------------------------------------------*/ class geomDecomp : public decompositionMethod { + // Private Member Functions + + //- Read input values and initialize the rotDelta_ + void readCoeffs(); + protected: // Protected data - const dictionary& geomDecomDict_; + //- Coefficients for all derived methods + const dictionary& coeffsDict_; Vector