From a9ffcab5af8df06e5e4af1cac326a1cda7c02316 Mon Sep 17 00:00:00 2001 From: Mark Olesen Date: Thu, 9 Nov 2017 12:30:24 +0100 Subject: [PATCH] ENH: region-wise decomposition specification for decomposeParDict Within decomposeParDict, it is now possible to specify a different decomposition method, methods coefficients or number of subdomains for each region individually. The top-level numberOfSubdomains remains mandatory, since this specifies the number of domains for the entire simulation. The individual regions may use the same number or fewer domains. Any optional method coefficients can be specified in a general "coeffs" entry or a method-specific one, eg "metisCoeffs". For multiLevel, only the method-specific "multiLevelCoeffs" dictionary is used, and is also mandatory. ---- ENH: shortcut specification for multiLevel. In addition to the longer dictionary form, it is also possible to use a shorter notation for multiLevel decomposition when the same decomposition method applies to each level. --- .../decomposePar/decomposePar.C | 10 +- .../decomposePar/decomposeParDict | 271 ++++++++----- .../decomposePar/domainDecomposition.C | 9 +- .../domainDecompositionDistribute.C | 6 +- .../preProcessing/mapFields/mapFields.C | 4 +- .../kahipDecomp/dummyKahipDecomp.C | 14 +- .../metisDecomp/dummyMetisDecomp.C | 14 +- .../ptscotchDecomp/dummyPtscotchDecomp.C | 16 +- .../scotchDecomp/dummyScotchDecomp.C | 14 +- .../decompose/decompose/decompositionModel.C | 1 + .../decompose/decompose/decompositionModel.H | 39 +- src/parallel/decompose/decomposition.dox | 282 +++++++++++++ .../decompositionMethod/decompositionMethod.C | 376 ++++++++++++++---- .../decompositionMethod/decompositionMethod.H | 135 ++++++- .../geomDecomp/geomDecomp.C | 70 +++- .../geomDecomp/geomDecomp.H | 41 +- .../hierarchGeomDecomp/hierarchGeomDecomp.C | 84 ++-- .../hierarchGeomDecomp/hierarchGeomDecomp.H | 32 +- .../manualDecomp/manualDecomp.C | 43 +- .../manualDecomp/manualDecomp.H | 26 +- .../metisLikeDecomp/metisLikeDecomp.C | 23 +- .../metisLikeDecomp/metisLikeDecomp.H | 31 +- .../multiLevelDecomp/multiLevelDecomp.C | 268 +++++++++++-- .../multiLevelDecomp/multiLevelDecomp.H | 22 +- .../decompositionMethods/noDecomp/noDecomp.C | 24 +- .../decompositionMethods/noDecomp/noDecomp.H | 32 +- .../simpleGeomDecomp/simpleGeomDecomp.C | 24 +- .../simpleGeomDecomp/simpleGeomDecomp.H | 25 +- .../structuredDecomp/structuredDecomp.C | 22 +- .../structuredDecomp/structuredDecomp.H | 10 +- .../decompose/kahipDecomp/kahipDecomp.C | 176 ++++---- .../decompose/kahipDecomp/kahipDecomp.H | 21 +- .../decompose/metisDecomp/metisDecomp.C | 37 +- .../decompose/metisDecomp/metisDecomp.H | 25 +- .../decompose/ptscotchDecomp/ptscotchDecomp.C | 155 ++++---- .../decompose/ptscotchDecomp/ptscotchDecomp.H | 16 +- .../decompose/scotchDecomp/scotchDecomp.C | 54 ++- .../decompose/scotchDecomp/scotchDecomp.H | 14 +- .../IO/fileHandler/system/decomposeParDict | 21 +- .../heatTransfer/system/decomposeParDict | 7 +- .../system/decomposeParDict | 25 +- .../system/decomposeParDict | 9 +- .../compartmentFire/system/decomposeParDict | 24 -- .../system/panelRegion/decomposeParDict | 23 +- .../system/decomposeParDict | 25 +- .../system/panelRegion/decomposeParDict | 21 +- .../smallPoolFire3D/system/decomposeParDict | 22 +- .../RAS/DLR_A_LTS/system/decomposeParDict | 26 +- .../system/decomposeParDict | 7 +- .../system/decomposeParDict | 9 +- .../system/decomposeParDict | 7 +- .../LadenburgJet60psi/system/decomposeParDict | 25 +- .../rutlandVortex2D/system/decomposeParDict | 9 +- .../RAS/squareBendLiq/system/decomposeParDict | 9 - .../system/decomposeParDict | 1 - .../injectorPipe/system/decomposeParDict | 9 +- .../squareBend/system/decomposeParDict | 11 - .../squareBendLiq/system/decomposeParDict | 11 - .../RAS/nacaAirfoil/system/decomposeParDict | 16 +- .../freeSpacePeriodic/system/decomposeParDict | 24 +- .../freeSpaceStream/system/decomposeParDict | 24 +- .../supersonicCorner/system/decomposeParDict | 25 +- .../wedge15Ma5/system/decomposeParDict | 25 +- .../periodicCubeArgon/system/decomposeParDict | 27 +- .../periodicCubeWater/system/decomposeParDict | 26 +- .../mdFoam/nanoNozzle/system/decomposeParDict | 27 +- .../iglooWithFridges/system/decomposeParDict | 25 +- .../system/bottomWater/decomposeParDict | 73 +--- .../system/decomposeParDict | 31 +- .../system/heater/decomposeParDict | 45 +-- .../system/leftSolid/decomposeParDict | 45 +-- .../system/rightSolid/decomposeParDict | 45 +-- .../system/topAir/decomposeParDict | 73 +--- .../system/air/decomposeParDict | 45 +-- .../externalSolarLoad/system/decomposeParDict | 22 +- .../system/floor/decomposeParDict | 45 +-- .../system/solid/decomposeParDict | 45 +-- .../multiRegionHeater/Allmesh | 26 ++ .../multiRegionHeater/Allrun | 23 +- .../multiRegionHeater/README.txt | 8 +- .../system/bottomWater/decomposeParDict | 73 +--- .../multiRegionHeater/system/decomposeParDict | 42 +- .../system/heater/decomposeParDict | 45 +-- .../system/leftSolid/decomposeParDict | 45 +-- .../system/rightSolid/decomposeParDict | 45 +-- .../system/topAir/decomposeParDict | 73 +--- .../system/bottomAir/decomposeParDict | 32 +- .../system/decomposeParDict | 32 +- .../system/heater/decomposeParDict | 31 +- .../system/leftSolid/decomposeParDict | 73 +--- .../system/rightSolid/decomposeParDict | 73 +--- .../system/topAir/decomposeParDict | 73 +--- .../system/cabin/decomposeParDict | 26 +- .../system/decomposeParDict | 4 - .../system/windshield/decomposeParDict | 26 +- .../system/cabin/decomposeParDict | 30 +- .../windshieldDefrost/system/decomposeParDict | 5 +- .../system/exterior/decomposeParDict | 30 +- .../system/ice/decomposeParDict | 30 +- .../heatExchanger/system/air/decomposeParDict | 25 +- .../heatExchanger/system/decomposeParDict | 25 +- .../system/porous/decomposeParDict | 25 +- .../jouleHeatingSolid/system/decomposeParDict | 2 - .../system/bottomAir/decomposeParDict | 45 +-- .../system/decomposeParDict | 36 +- .../system/heater/decomposeParDict | 47 +-- .../system/leftSolid/decomposeParDict | 47 +-- .../system/rightSolid/decomposeParDict | 47 +-- .../system/topAir/decomposeParDict | 45 +-- .../system/coarseMesh/decomposeParDict | 2 - .../cavityMappingTest/system/decomposeParDict | 2 - .../building/steady/system/decomposeParDict | 9 +- .../system/decomposeParDict | 8 +- .../simpleRotor/system/decomposeParDict | 7 +- .../twoSimpleRotors/system/decomposeParDict | 8 +- .../system/decomposeParDict | 19 +- .../system/decomposeParDict | 19 +- .../mixerVesselAMI2D/system/decomposeParDict | 1 - .../system/decomposeParDict | 10 +- .../propeller/system/decomposeParDict | 10 +- .../system/decomposeParDict | 16 +- .../system/decomposeParDict | 20 +- .../LES/channel395/system/decomposeParDict | 112 +----- .../channel395DFSEM/system/decomposeParDict | 7 +- .../LES/vortexShed/system/decomposeParDict | 7 +- .../motorBike/system/decomposeParDict | 21 +- .../system/decomposeParDict.hierarchical | 20 +- .../system/decomposeParDict.ptscotch | 21 +- .../pitzDailyMapped/system/decomposeParDict | 24 -- .../system/decomposeParDict | 20 +- .../motorBike/system/decomposeParDict.6 | 22 +- .../pipeCyclic/system/decomposeParDict | 1 - .../system/decomposeParDict | 21 +- .../system/decomposeParDict.hierarchical | 6 +- .../Goldschmidt/system/decomposeParDict | 7 +- .../Goldschmidt/system/decomposeParDict | 7 +- .../MPPICFoam/cyclone/system/decomposeParDict | 8 +- .../simplifiedSiwek/system/decomposeParDict | 20 +- .../hopperEmptying/system/decomposeParDict | 19 +- .../system/decomposeParDict | 20 +- .../cylinder/system/decomposeParDict | 21 +- .../filter/system/decomposeParDict | 22 +- .../hotBoxes/system/decomposeParDict | 1 - .../wallFilmRegion.orig/decomposeParDict | 1 - .../rivuletPanel/system/decomposeParDict | 21 +- .../splashPanel/system/decomposeParDict | 21 +- .../decomposeParDict | 21 +- .../foamyHexMesh/blob/system/decomposeParDict | 25 +- .../decomposeParDict | 21 +- .../flange/system/decomposeParDict | 21 +- .../constant/triSurface/surfaceProcess.sh | 5 +- .../mixerVessel/system/decomposeParDict | 25 +- .../OpenCFD/system/decomposeParDict | 20 +- .../SnakeRiverCanyon/system/decomposeParDict | 19 +- .../system/decomposeParDict | 9 +- .../box_snappyHexMesh/system/decomposeParDict | 9 +- .../parallel/filter/system/decomposeParDict | 19 - .../flange/system/decomposeParDict | 20 +- .../gap_detection/system/decomposeParDict | 23 +- .../twoPhasePachuka/system/decomposeParDict | 9 +- .../LES/throttle3D/system/decomposeParDict | 20 +- .../sloshingTank2D/system/decomposeParDict | 20 +- .../sphereDrop/system/decomposeParDict | 5 +- .../depthCharge3D/system/decomposeParDict | 20 +- .../damBreak4phase/system/decomposeParDict | 20 +- .../RAS/DTCHull/system/decomposeParDict | 38 +- .../floatingObject/system/decomposeParDict | 23 -- .../mixerVesselAMI/system/decomposeParDict | 1 - .../RAS/motorBike/system/decomposeParDict | 21 +- .../system/decomposeParDict | 24 -- .../sloshingTank2D/system/decomposeParDict | 20 +- .../system/decomposeParDict | 20 +- .../sloshingTank3D/system/decomposeParDict | 20 +- .../system/decomposeParDict | 21 +- .../system/decomposeParDict | 21 +- .../testTubeMixer/system/decomposeParDict | 21 +- .../RAS/DTCHull/system/decomposeParDict | 32 +- .../damBreak/damBreak/system/decomposeParDict | 20 +- .../damBreak/damBreak/system/decomposeParDict | 19 +- .../eulerianInjection/system/decomposeParDict | 6 +- .../system/decomposeParDict | 5 +- .../system/decomposeParDict | 7 +- .../system/decomposeParDict | 6 +- .../system/decomposeParDict | 6 +- .../system/decomposeParDict | 6 +- .../system/decomposeParDict | 6 +- .../system/decomposeParDict | 6 +- .../system/decomposeParDict | 6 +- .../system/decomposeParDict | 8 +- .../damBreak/system/decomposeParDict | 20 +- .../system/decomposeParDict | 10 +- .../system/decomposeParDict | 13 +- .../standingWave/system/decomposeParDict | 20 +- .../laminar/damBreak/system/decomposeParDict | 20 +- .../propeller/system/decomposeParDict | 8 +- .../cavitatingBullet/system/decomposeParDict | 21 +- .../damBreak4phase/system/decomposeParDict | 21 +- .../system/decomposeParDict | 21 +- .../mixerVesselAMI2D/system/decomposeParDict | 1 - .../damBreak4phase/system/decomposeParDict | 21 +- .../system/decomposeParDict | 21 +- .../background/system/decomposeParDict | 7 +- .../system/decomposeParDict | 7 +- .../system/decomposeParDict | 7 +- .../motorBike/system/decomposeParDict | 20 +- .../system/bottomAir/decomposeParDict | 32 +- .../system/decomposeParDict | 32 +- .../system/heater/decomposeParDict | 32 +- .../system/leftSolid/decomposeParDict | 32 +- .../system/rightSolid/decomposeParDict | 32 +- .../system/topAir/decomposeParDict | 32 +- 211 files changed, 2416 insertions(+), 3933 deletions(-) create mode 100644 src/parallel/decompose/decomposition.dox mode change 100644 => 120000 tutorials/heatTransfer/chtMultiRegionFoam/externalCoupledMultiRegionHeater/system/bottomWater/decomposeParDict mode change 100644 => 120000 tutorials/heatTransfer/chtMultiRegionFoam/externalCoupledMultiRegionHeater/system/heater/decomposeParDict mode change 100644 => 120000 tutorials/heatTransfer/chtMultiRegionFoam/externalCoupledMultiRegionHeater/system/leftSolid/decomposeParDict mode change 100644 => 120000 tutorials/heatTransfer/chtMultiRegionFoam/externalCoupledMultiRegionHeater/system/rightSolid/decomposeParDict mode change 100644 => 120000 tutorials/heatTransfer/chtMultiRegionFoam/externalCoupledMultiRegionHeater/system/topAir/decomposeParDict mode change 100644 => 120000 tutorials/heatTransfer/chtMultiRegionFoam/externalSolarLoad/system/air/decomposeParDict mode change 100644 => 120000 tutorials/heatTransfer/chtMultiRegionFoam/externalSolarLoad/system/floor/decomposeParDict mode change 100644 => 120000 tutorials/heatTransfer/chtMultiRegionFoam/externalSolarLoad/system/solid/decomposeParDict create mode 100755 tutorials/heatTransfer/chtMultiRegionFoam/multiRegionHeater/Allmesh mode change 100644 => 120000 tutorials/heatTransfer/chtMultiRegionFoam/multiRegionHeater/system/bottomWater/decomposeParDict mode change 100644 => 120000 tutorials/heatTransfer/chtMultiRegionFoam/multiRegionHeater/system/heater/decomposeParDict mode change 100644 => 120000 tutorials/heatTransfer/chtMultiRegionFoam/multiRegionHeater/system/leftSolid/decomposeParDict mode change 100644 => 120000 tutorials/heatTransfer/chtMultiRegionFoam/multiRegionHeater/system/rightSolid/decomposeParDict mode change 100644 => 120000 tutorials/heatTransfer/chtMultiRegionFoam/multiRegionHeater/system/topAir/decomposeParDict mode change 100644 => 120000 tutorials/heatTransfer/chtMultiRegionFoam/snappyMultiRegionHeater/system/leftSolid/decomposeParDict mode change 100644 => 120000 tutorials/heatTransfer/chtMultiRegionFoam/snappyMultiRegionHeater/system/rightSolid/decomposeParDict mode change 100644 => 120000 tutorials/heatTransfer/chtMultiRegionFoam/snappyMultiRegionHeater/system/topAir/decomposeParDict mode change 100644 => 120000 tutorials/heatTransfer/chtMultiRegionFoam/windshieldCondensation/system/cabin/decomposeParDict mode change 100644 => 120000 tutorials/heatTransfer/chtMultiRegionFoam/windshieldCondensation/system/windshield/decomposeParDict mode change 100644 => 120000 tutorials/heatTransfer/chtMultiRegionFoam/windshieldDefrost/system/cabin/decomposeParDict mode change 100644 => 120000 tutorials/heatTransfer/chtMultiRegionFoam/windshieldDefrost/system/exterior/decomposeParDict mode change 100644 => 120000 tutorials/heatTransfer/chtMultiRegionFoam/windshieldDefrost/system/ice/decomposeParDict mode change 100644 => 120000 tutorials/heatTransfer/chtMultiRegionSimpleFoam/multiRegionHeaterRadiation/system/bottomAir/decomposeParDict mode change 100644 => 120000 tutorials/heatTransfer/chtMultiRegionSimpleFoam/multiRegionHeaterRadiation/system/heater/decomposeParDict mode change 100644 => 120000 tutorials/heatTransfer/chtMultiRegionSimpleFoam/multiRegionHeaterRadiation/system/leftSolid/decomposeParDict mode change 100644 => 120000 tutorials/heatTransfer/chtMultiRegionSimpleFoam/multiRegionHeaterRadiation/system/rightSolid/decomposeParDict mode change 100644 => 120000 tutorials/heatTransfer/chtMultiRegionSimpleFoam/multiRegionHeaterRadiation/system/topAir/decomposeParDict diff --git a/applications/utilities/parallelProcessing/decomposePar/decomposePar.C b/applications/utilities/parallelProcessing/decomposePar/decomposePar.C index b06da4f4a7..d1b9b58a7d 100644 --- a/applications/utilities/parallelProcessing/decomposePar/decomposePar.C +++ b/applications/utilities/parallelProcessing/decomposePar/decomposePar.C @@ -330,7 +330,6 @@ int main(int argc, char *argv[]) } - forAll(regionNames, regioni) { const word& regionName = regionNames[regioni]; @@ -338,13 +337,12 @@ int main(int argc, char *argv[]) Info<< "\n\nDecomposing mesh " << regionName << nl << endl; - // Determine the existing processor count directly label nProcs = fileHandler().nProcs(runTime.path(), regionDir); - // Get requested numberOfSubdomains. Note: have no mesh yet so - // cannot use decompositionModel::New - const label nDomains = readLabel + // Get requested numberOfSubdomains directly from the dictionary. + // Note: have no mesh yet so cannot use decompositionModel::New + const label nDomains = decompositionMethod::nDomains ( IOdictionary ( @@ -362,7 +360,7 @@ int main(int argc, char *argv[]) ), decompDictFile ) - ).lookup("numberOfSubdomains") + ) ); if (decomposeFieldsOnly) diff --git a/applications/utilities/parallelProcessing/decomposePar/decomposeParDict b/applications/utilities/parallelProcessing/decomposePar/decomposeParDict index cb061078b7..2ad99e2ea5 100644 --- a/applications/utilities/parallelProcessing/decomposePar/decomposeParDict +++ b/applications/utilities/parallelProcessing/decomposePar/decomposeParDict @@ -15,10 +15,174 @@ FoamFile } // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // -numberOfSubdomains 2; +//- The total number of domains (mandatory) +numberOfSubdomains 256; + +//- The decomposition method (mandatory) +method scotch; +// method hierarchical; +// method simple; +// method metis; +// method manual; +// method multiLevel; +// method structured; // does 2D decomposition of structured mesh -// Optional decomposition constraints +//- Optional region-wise decomposition. +// Can specify a different method. +// The number of subdomains can be less than the top-level numberOfSubdomains. +regions +{ + water + { + numberOfSubdomains 128; + method metis; + } + + ".*solid" + { + numberOfSubdomains 4; + method metis; + } + + heater + { + numberOfSubdomains 1; + method none; + } +} + + +// Coefficients for the decomposition method are either as a +// general "coeffs" dictionary or method-specific "Coeffs". +// For multiLevel, using multiLevelCoeffs only. + + +multiLevelCoeffs +{ + // multiLevel decomposition methods to apply in turn. + // This is like hierarchical but fully general + // - every method can be used at every level. + + // Only sub-dictionaries containing the keyword "method" are used. + // + + level0 + { + numberOfSubdomains 16; + method scotch; + } + level1 + { + numberOfSubdomains 2; + method scotch; + coeffs + { + n (2 1 1); + delta 0.001; + } + } + level2 + { + numberOfSubdomains 8; + // method simple; + method scotch; + } +} + + +multiLevelCoeffs +{ + // Compact multiLevel specification, activated by the presence of the + // keywords "method" and "domains" + + method scotch; + domains (16 2 8); + + //// Or with implicit '16' for the first level with numberOfSubdomains=256 + //domains (2 8); +} + + + +// Other example coefficents + +simpleCoeffs +{ + n (2 1 1); + // delta 0.001; //< default value = 0.001 +} + +hierarchicalCoeffs +{ + n (1 2 1); + // delta 0.001; //< default value = 0.001 + // order xyz; //< default order = xyz +} + +metisCoeffs +{ + /* + processorWeights + ( + 1 + 1 + 1 + 1 + ); + */ +} + +scotchCoeffs +{ + //processorWeights + //( + // 1 + // 1 + // 1 + // 1 + //); + //writeGraph true; + //strategy "b"; +} + +manualCoeffs +{ + dataFile "decompositionData"; +} + +structuredCoeffs +{ + // Patches to do 2D decomposition on. Structured mesh only; cells have + // to be in 'columns' on top of patches. + patches (movingWall); + + // Method to use on the 2D subset + method scotch; +} + + +//- Use the volScalarField named here as a weight for each cell in the +// decomposition. For example, use a particle population field to decompose +// for a balanced number of particles in a lagrangian simulation. +// weightField dsmcRhoNMean; + + +//// Is the case distributed? Note: command-line argument -roots takes +//// precedence +//distributed yes; +// +//// Per slave (so nProcs-1 entries) the directory above the case. +//roots +//( +// "/tmp" +// "/tmp" +//); + + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // +// Decomposition constraints + //constraints //{ // preserveBaffles @@ -88,107 +252,4 @@ numberOfSubdomains 2; //preserveBaffles true; - -//- Use the volScalarField named here as a weight for each cell in the -// decomposition. For example, use a particle population field to decompose -// for a balanced number of particles in a lagrangian simulation. -// weightField dsmcRhoNMean; - -method scotch; -//method hierarchical; -// method simple; -// method metis; -// method manual; -// method multiLevel; -// method structured; // does 2D decomposition of structured mesh - -multiLevelCoeffs -{ - // Decomposition methods to apply in turn. This is like hierarchical but - // fully general - every method can be used at every level. - - level0 - { - numberOfSubdomains 64; - //method simple; - //simpleCoeffs - //{ - // n (2 1 1); - // delta 0.001; - //} - method scotch; - } - level1 - { - numberOfSubdomains 4; - method scotch; - } -} - -// Desired output - -simpleCoeffs -{ - n (2 1 1); - delta 0.001; -} - -hierarchicalCoeffs -{ - n (1 2 1); - delta 0.001; - order xyz; -} - -metisCoeffs -{ - /* - processorWeights - ( - 1 - 1 - 1 - 1 - ); - */ -} - -scotchCoeffs -{ - //processorWeights - //( - // 1 - // 1 - // 1 - // 1 - //); - //writeGraph true; - //strategy "b"; -} - -manualCoeffs -{ - dataFile "decompositionData"; -} - -structuredCoeffs -{ - // Patches to do 2D decomposition on. Structured mesh only; cells have - // to be in 'columns' on top of patches. - patches (movingWall); - - // Method to use on the 2D subset - method scotch; -} - -//// Is the case distributed? Note: command-line argument -roots takes -//// precedence -//distributed yes; -//// Per slave (so nProcs-1 entries) the directory above the case. -//roots -//( -// "/tmp" -// "/tmp" -//); - // ************************************************************************* // diff --git a/applications/utilities/parallelProcessing/decomposePar/domainDecomposition.C b/applications/utilities/parallelProcessing/decomposePar/domainDecomposition.C index bf722cfbae..0cc795761c 100644 --- a/applications/utilities/parallelProcessing/decomposePar/domainDecomposition.C +++ b/applications/utilities/parallelProcessing/decomposePar/domainDecomposition.C @@ -49,10 +49,8 @@ void Foam::domainDecomposition::mark labelList& elementToZone ) { - forAll(zoneElems, i) + for (const label pointi : zoneElems) { - label pointi = zoneElems[i]; - if (elementToZone[pointi] == -1) { // First occurrence @@ -69,7 +67,6 @@ void Foam::domainDecomposition::mark // * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * // -// from components Foam::domainDecomposition::domainDecomposition ( const IOobject& io, @@ -98,13 +95,13 @@ Foam::domainDecomposition::domainDecomposition decompDictFile_(decompDictFile), nProcs_ ( - readInt + decompositionMethod::nDomains ( decompositionModel::New ( *this, decompDictFile - ).lookup("numberOfSubdomains") + ) ) ), distributed_(false), diff --git a/applications/utilities/parallelProcessing/decomposePar/domainDecompositionDistribute.C b/applications/utilities/parallelProcessing/decomposePar/domainDecompositionDistribute.C index 371f44f73e..472077a709 100644 --- a/applications/utilities/parallelProcessing/decomposePar/domainDecompositionDistribute.C +++ b/applications/utilities/parallelProcessing/decomposePar/domainDecompositionDistribute.C @@ -46,11 +46,11 @@ void Foam::domainDecomposition::distributeCells() decompDictFile_ ); + word weightName; scalarField cellWeights; - if (method.found("weightField")) - { - word weightName = method.lookup("weightField"); + if (method.readIfPresent("weightField", weightName)) + { volScalarField weights ( IOobject diff --git a/applications/utilities/preProcessing/mapFields/mapFields.C b/applications/utilities/preProcessing/mapFields/mapFields.C index 93ba316412..be6261d3f0 100644 --- a/applications/utilities/preProcessing/mapFields/mapFields.C +++ b/applications/utilities/preProcessing/mapFields/mapFields.C @@ -58,7 +58,7 @@ int readNumProcs dictFile = dictFile / dictName; } - return readInt + return decompositionMethod::nDomains ( IOdictionary ( @@ -75,7 +75,7 @@ int readNumProcs ), dictFile ) - ).lookup("numberOfSubdomains") + ) ); } diff --git a/src/dummyThirdParty/kahipDecomp/dummyKahipDecomp.C b/src/dummyThirdParty/kahipDecomp/dummyKahipDecomp.C index 96e0bdc16b..6e147e2626 100644 --- a/src/dummyThirdParty/kahipDecomp/dummyKahipDecomp.C +++ b/src/dummyThirdParty/kahipDecomp/dummyKahipDecomp.C @@ -73,10 +73,20 @@ Foam::label Foam::kahipDecomp::decomposeSerial Foam::kahipDecomp::kahipDecomp ( - const dictionary& decompositionDict + const dictionary& decompDict ) : - metisLikeDecomp(decompositionDict) + metisLikeDecomp("kahip", decompDict) +{} + + +Foam::kahipDecomp::kahipDecomp +( + const dictionary& decompDict, + const word& regionName +) +: + metisLikeDecomp("kahip", decompDict, regionName) {} diff --git a/src/dummyThirdParty/metisDecomp/dummyMetisDecomp.C b/src/dummyThirdParty/metisDecomp/dummyMetisDecomp.C index 3743cc1bae..c5f8e3ba9a 100644 --- a/src/dummyThirdParty/metisDecomp/dummyMetisDecomp.C +++ b/src/dummyThirdParty/metisDecomp/dummyMetisDecomp.C @@ -73,10 +73,20 @@ Foam::label Foam::metisDecomp::decomposeSerial Foam::metisDecomp::metisDecomp ( - const dictionary& decompositionDict + const dictionary& decompDict ) : - metisLikeDecomp(decompositionDict) + metisLikeDecomp("metis", decompDict) +{} + + +Foam::metisDecomp::metisDecomp +( + const dictionary& decompDict, + const word& regionName +) +: + metisLikeDecomp("metis", decompDict, regionName) {} diff --git a/src/dummyThirdParty/ptscotchDecomp/dummyPtscotchDecomp.C b/src/dummyThirdParty/ptscotchDecomp/dummyPtscotchDecomp.C index decacb6c10..5a756b477f 100644 --- a/src/dummyThirdParty/ptscotchDecomp/dummyPtscotchDecomp.C +++ b/src/dummyThirdParty/ptscotchDecomp/dummyPtscotchDecomp.C @@ -98,10 +98,22 @@ Foam::label Foam::ptscotchDecomp::decompose Foam::ptscotchDecomp::ptscotchDecomp ( - const dictionary& decompositionDict + const dictionary& decompDict ) : - decompositionMethod(decompositionDict) + decompositionMethod(decompDict), + coeffsDict_(dictionary::null) +{} + + +Foam::ptscotchDecomp::ptscotchDecomp +( + const dictionary& decompDict, + const word& regionName +) +: + decompositionMethod(decompDict, regionName), + coeffsDict_(dictionary::null) {} diff --git a/src/dummyThirdParty/scotchDecomp/dummyScotchDecomp.C b/src/dummyThirdParty/scotchDecomp/dummyScotchDecomp.C index 582c770253..9aa8e20ec0 100644 --- a/src/dummyThirdParty/scotchDecomp/dummyScotchDecomp.C +++ b/src/dummyThirdParty/scotchDecomp/dummyScotchDecomp.C @@ -80,10 +80,20 @@ Foam::label Foam::scotchDecomp::decomposeSerial Foam::scotchDecomp::scotchDecomp ( - const dictionary& decompositionDict + const dictionary& decompDict ) : - metisLikeDecomp(decompositionDict) + metisLikeDecomp("scotch", decompDict) +{} + + +Foam::scotchDecomp::scotchDecomp +( + const dictionary& decompDict, + const word& regionName +) +: + metisLikeDecomp("scotch", decompDict, regionName) {} diff --git a/src/parallel/decompose/decompose/decompositionModel.C b/src/parallel/decompose/decompose/decompositionModel.C index dbcb023af1..8f156c64b5 100644 --- a/src/parallel/decompose/decompose/decompositionModel.C +++ b/src/parallel/decompose/decompose/decompositionModel.C @@ -34,6 +34,7 @@ namespace Foam defineTypeNameAndDebug(decompositionModel, 0); } + // * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * // Foam::decompositionModel::decompositionModel diff --git a/src/parallel/decompose/decompose/decompositionModel.H b/src/parallel/decompose/decompose/decompositionModel.H index 7ed6bd3fa2..2cbfa4d249 100644 --- a/src/parallel/decompose/decompose/decompositionModel.H +++ b/src/parallel/decompose/decompose/decompositionModel.H @@ -75,14 +75,14 @@ public: // Selectors - //- Read (optionallly from absolute path) & register on mesh + //- Read (optionally from absolute path) & register on mesh static const decompositionModel& New ( const polyMesh& mesh, const fileName& decompDictFile = "" ); - //- Read (optionallly from supplied dictionary) & register on mesh + //- Read (optionally from supplied dictionary) & register on mesh static const decompositionModel& New ( const polyMesh& mesh, @@ -94,42 +94,51 @@ public: // Constructors //- Construct from typeName or optional path to controlDictionary - decompositionModel(const polyMesh&, const fileName& = ""); + decompositionModel + ( + const polyMesh& mesh, + const fileName& decompDictFile = "" + ); //- Construct from typeName or optional path to controlDictionary decompositionModel ( - const polyMesh&, + const polyMesh& mesh, const dictionary& dict, - const fileName& = "" + const fileName& decompDictFile = "" ); - // Member functions + // Member Functions decompositionMethod& decomposer() const { if (!decomposerPtr_.valid()) { - decomposerPtr_ = decompositionMethod::New(*this); + decomposerPtr_ = + decompositionMethod::New + ( + *this, + this->mesh().name() // Name of mesh region + ); } return decomposerPtr_(); } //- Helper: return IOobject with optionally absolute path provided - static IOobject selectIO(const IOobject&, const fileName&); + static IOobject selectIO(const IOobject& io, const fileName& f); - // UpdateableMeshObject + // UpdateableMeshObject - virtual bool movePoints() - { - return false; - } + virtual bool movePoints() + { + return false; + } - virtual void updateMesh(const mapPolyMesh&) - {} + virtual void updateMesh(const mapPolyMesh&) + {} }; diff --git a/src/parallel/decompose/decomposition.dox b/src/parallel/decompose/decomposition.dox new file mode 100644 index 0000000000..0256b48e4a --- /dev/null +++ b/src/parallel/decompose/decomposition.dox @@ -0,0 +1,282 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | OpenFOAM: The Open Source CFD Toolbox + \\ / O peration | + \\ / A nd | Copyright (C) 2017 OpenCFD Ltd. + \\/ M anipulation | +------------------------------------------------------------------------------- +License + This file is part of OpenFOAM. + + OpenFOAM is free software: you can redistribute it and/or modify it under + the terms of the GNU General Public License as published by the Free + Software Foundation, either version 3 of the License, or (at your option) + any later version. + + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + details. + + You should have received a copy of the GNU General Public License along with + OpenFOAM. If not, see . + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +\page domainDecomposition Domain decomposition + +\section secIntroduction Introduction + +When running a simulation in parallel, the geometry must first be decomposed +(segmented) into individual geometries for each MPI process. These separate +geometries are \a connected together with special processor boundary patches. +The processor-specific \c constant/polyMesh/boundary files will contain this +type of entry: + +\verbatim + procBoundary0to14 + { + type processor; + inGroups 1(processor); + nFaces 131; + startFace 34983; + matchTolerance 0.0001; + transform unknown; + myProcNo 0; + neighbProcNo 14; + } +\endverbatim + +The decomposePar utility is a commonly used method to decompose domains +and subsequently distribute the fields. +The reconstructPar and reconstructParMesh utilities +can be used to reconstruct a single domain from the processor sub-domains. + +\section secDecomposeParDict The decomposeParDict + +The \c decomposeParDict is required by decompose utilities and for any solvers +or utilities running in parallel. It is normally located in the simulation +\c system directory. The -decomposeParDict name command-line option +can be used to specify an alternate file. + +The \a numberOfSubdomains entry is mandatory: +\verbatim + numberOfSubdomains ; +\endverbatim + +The \a method entry is required for the decomposePar utility and specifies +the decomposition method type: +\verbatim + method ; +\endverbatim +The \a method entry is generally not required when running a simulation. + +OpenFOAM offers a variety of decomposition methods and interfaces to external, +third-party decomposition routines. The types of decomposition methods +available will thus depend on your particular installation. + +\table + Name | Class + none | #Foam::noDecomp + manual | #Foam::manualDecomp + simple | #Foam::simpleGeomDecomp + hierarchical | #Foam::hierarchGeomDecomp + kahip | #Foam::kahipDecomp + metis | #Foam::metisDecomp + scotch | #Foam::scotchDecomp + structured | #Foam::structuredDecomp + multiLevel | #Foam::multiLevelDecomp +\endtable + +If a decomposition method requires any additional configuration controls, +these are specified either within in a generic \c coeffs dictionary that +or a method-specific version. For example, + +\verbatim + method hierarchical; + + coeffs + { + n (4 2 3); + } + + // ----- + + method metis; + + metisCoeffs + { + method k-way; + } +\endverbatim + +For simplicity, the generic \c coeffs dictionary is generally preferrable. +However, for some specific decomposition methods +(eg, \ref subsecMultiLevel "multiLevel") only the +method-specific coefficients dictionary is permitted. + + +\subsection subsecRegions Multi-region + +When running multi-region simulations, it may be desirable to use different +decomposition methods for one or more regions, or even to have fewer +processors allocated to a particular region. +If, for example, the multi-region simulation contains a large fluid region +and a very small solid region, it can be advantageous to decompose the solid +onto fewer processors. + +The region-wise specification is contained in a \a regions subdictionary with +decomposeParDict. For example, + +\verbatim + numberOfSubdomains 2048; + method metis; + + regions + { + heater + { + numberOfSubdomains 2; + method hierarchical; + coeffs + { + n (2 1 1); + } + } + + "*.solid" + { + numberOfSubdomains 16; + method scotch; + } + } +\endverbatim + +\note +The top-level numberOfSubdomains remains mandatory, since this specifies the +number of domains for the entire simulation. The individual regions may use +the same number or fewer domains. The \a numberOfSubdomains entry within +a region specification is only needed if the value differs. + + +\subsection subsecMultiLevel Multi-level decomposition + +The #Foam::multiLevelDecomp decomposition provides a general means of +successively decomposing with different methods. Each appplication of the +decomposition is termed a level. For example, + +\verbatim + numberOfSubdomains 2048; + method multiLevel; + + multiLevelCoeffs + { + nodes + { + numberOfSubdomains 128; + method hierarchical; + coeffs + { + n (16 4 2); + } + } + cpus + { + numberOfSubdomains 2; + method scotch; + } + cores + { + numberOfSubdomains 8; + method metis; + } + } +\endverbatim + +For cases where the same method is applied at each level, this can also be +conveniently written in a much shorter form: +\verbatim + numberOfSubdomains 2048; + method multiLevel; + + multiLevelCoeffs + { + method scotch + domains (128 2 8); + } +\endverbatim + +When the specified \a domains is smaller than \a numberOfSubdomains +but can be resolved as an integral multiple, this integral multiple +is used as the first level. This can make it easier to manage when +changing the number of domains for the simulation. +For example, +\verbatim + numberOfSubdomains 1024; + method multiLevel; + + multiLevelCoeffs + { + method scotch + domains (2 8); //< inferred as domains (64 2 8); + } +\endverbatim + + +\subsection subsecConstraints Constraints + +\verbatim + constraints + { + preserveBaffles + { + // Keep owner and neighbour of baffles on same processor + // (ie, keep it detectable as a baffle). + // Baffles are two boundary face sharing the same points + + type preserveBaffles; + } + + preserveFaceZones + { + // Keep owner and neighbour on same processor for faces in zones + + type preserveFaceZones; + zones (".*"); + } + + preservePatches + { + // Keep owner and neighbour on same processor for faces in patches + // (only makes sense for cyclic patches. Not suitable for e.g. + // cyclicAMI since these are not coupled on the patch level. + // Use singleProcessorFaceSets for those. + + type preservePatches; + patches (".*"); + } + + singleProcessorFaceSets + { + // Keep all of faceSet on a single processor. This puts all cells + // connected with a point, edge or face on the same processor. + // (just having face connected cells might not guarantee a balanced + // decomposition) + // The processor can be -1 (the decompositionMethod chooses the + // processor for a good load balance) or explicitly provided (upsets + // balance) + + type singleProcessorFaceSets; + singleProcessorFaceSets ((f1 -1)); + } + + refinementHistory + { + // Decompose cells such that all cell originating from single cell + // end up on same processor + type refinementHistory; + } + } +\endverbatim + +\*---------------------------------------------------------------------------*/ diff --git a/src/parallel/decompose/decompositionMethods/decompositionMethod/decompositionMethod.C b/src/parallel/decompose/decompositionMethods/decompositionMethod/decompositionMethod.C index c17958367a..69bc26b4d3 100644 --- a/src/parallel/decompose/decompositionMethods/decompositionMethod/decompositionMethod.C +++ b/src/parallel/decompose/decompositionMethods/decompositionMethod/decompositionMethod.C @@ -44,49 +44,97 @@ namespace Foam { defineTypeNameAndDebug(decompositionMethod, 0); defineRunTimeSelectionTable(decompositionMethod, dictionary); + defineRunTimeSelectionTable(decompositionMethod, dictionaryRegion); + + // Fallback name when searching for optional coefficients directories + static const word defaultName("coeffs"); } -// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * // +// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * // -Foam::decompositionMethod::decompositionMethod -( - const dictionary& decompositionDict -) -: - decompositionDict_(decompositionDict), - nProcessors_ - ( - readLabel(decompositionDict.lookup("numberOfSubdomains")) - ) +Foam::label Foam::decompositionMethod::nDomains(const dictionary& decompDict) { - // Read any constraints - wordList constraintTypes_; - if (decompositionDict_.found("constraints")) + return readLabel(decompDict.lookup("numberOfSubdomains")); +} + + +Foam::label Foam::decompositionMethod::nDomains +( + const dictionary& decompDict, + const word& regionName +) +{ + const label nDomainsGlobal = nDomains(decompDict); + + const dictionary& regionDict(optionalRegionDict(decompDict, regionName)); + + label nDomainsRegion; + if (regionDict.readIfPresent("numberOfSubdomains", nDomainsRegion)) { - //PtrList constraintsList - //( - // decompositionDict_.lookup("constraints") - //); - //forAll(constraintsList, i) - //{ - // const dictionary& dict = constraintsList[i]; - const dictionary& constraintsList = decompositionDict_.subDict - ( - "constraints" - ); - forAllConstIter(dictionary, constraintsList, iter) + if (nDomainsRegion >= 1 && nDomainsRegion <= nDomainsGlobal) + { + return nDomainsRegion; + } + + WarningInFunction + << "ignoring out of range numberOfSubdomains " + << nDomainsRegion << " for region " << regionName + << nl << nl + << endl; + } + + return nDomainsGlobal; +} + + +const Foam::dictionary& Foam::decompositionMethod::optionalRegionDict +( + const dictionary& decompDict, + const word& regionName +) +{ + auto finder = decompDict.csearch("regions"); + + if (!regionName.empty() && finder.isDict()) + { + finder = finder.dict().csearch(regionName); + + if (finder.isDict()) + { + return finder.dict(); + } + } + + return dictionary::null; +} + + +// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * // + +void Foam::decompositionMethod::readConstraints() +{ + constraints_.clear(); + + // Read any constraints + wordList constraintTypes; + + const dictionary* dictptr = decompositionDict_.subDictPtr("constraints"); + + if (dictptr) + { + forAllConstIters(*dictptr, iter) { const dictionary& dict = iter().dict(); - constraintTypes_.append(dict.lookup("type")); + constraintTypes.append(dict.lookup("type")); constraints_.append ( decompositionConstraint::New ( dict, - constraintTypes_.last() + constraintTypes.last() ) ); } @@ -96,7 +144,7 @@ Foam::decompositionMethod::decompositionMethod if ( decompositionDict_.found("preserveBaffles") - && !constraintTypes_.found + && !constraintTypes.found ( decompositionConstraints::preserveBafflesConstraint::typeName ) @@ -111,7 +159,7 @@ Foam::decompositionMethod::decompositionMethod if ( decompositionDict_.found("preservePatches") - && !constraintTypes_.found + && !constraintTypes.found ( decompositionConstraints::preservePatchesConstraint::typeName ) @@ -128,7 +176,7 @@ Foam::decompositionMethod::decompositionMethod if ( decompositionDict_.found("preserveFaceZones") - && !constraintTypes_.found + && !constraintTypes.found ( decompositionConstraints::preserveFaceZonesConstraint::typeName ) @@ -145,7 +193,7 @@ Foam::decompositionMethod::decompositionMethod if ( decompositionDict_.found("singleProcessorFaceSets") - && !constraintTypes_.found + && !constraintTypes.found ( decompositionConstraints::preserveFaceZonesConstraint::typeName ) @@ -166,17 +214,144 @@ Foam::decompositionMethod::decompositionMethod } } +// * * * * * * * * * * * * Protected Member Functions * * * * * * * * * * * // + +const Foam::dictionary& Foam::decompositionMethod::findCoeffsDict +( + const dictionary& dict, + const word& coeffsName, + int select +) +{ + dictionary::const_searcher fnd; + + if + ( + (fnd = dict.csearch(coeffsName)).isDict() + || + ( + !(select & selectionType::EXACT) + && (fnd = dict.csearch(defaultName)).isDict() + ) + ) + { + return fnd.dict(); + } + + // Not found + if (select & selectionType::MANDATORY) + { + FatalIOError + << "'" << coeffsName << "' dictionary not found in dictionary " + << dict.name() << endl + << abort(FatalIOError); + } + + if (select & selectionType::NULL_DICT) + { + return dictionary::null; + } + + return dict; +} + + +const Foam::dictionary& Foam::decompositionMethod::findCoeffsDict +( + const word& coeffsName, + int select +) const +{ + dictionary::const_searcher fnd; + + if + ( + !decompositionRegionDict_.empty() + && + ( + (fnd = decompositionRegionDict_.csearch(coeffsName)).isDict() + || + ( + !(select & selectionType::EXACT) + && (fnd = decompositionRegionDict_.csearch(defaultName)).isDict() + ) + ) + ) + { + return fnd.dict(); + } + + if + ( + (fnd = decompositionDict_.csearch(coeffsName)).isDict() + || + ( + !(select & selectionType::EXACT) + && (fnd = decompositionDict_.csearch(defaultName)).isDict() + ) + ) + { + return fnd.dict(); + } + + // Not found + if (select & selectionType::MANDATORY) + { + FatalIOError + << "'" << coeffsName << "' dictionary not found in dictionary " + << decompositionDict_.name() << endl + << abort(FatalIOError); + } + + if (select & selectionType::NULL_DICT) + { + return dictionary::null; + } + + return decompositionDict_; +} + + +// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * // + +Foam::decompositionMethod::decompositionMethod +( + const dictionary& decompDict +) +: + decompositionDict_(decompDict), + decompositionRegionDict_(dictionary::null), + nDomains_(nDomains(decompDict)) +{ + readConstraints(); +} + + +Foam::decompositionMethod::decompositionMethod +( + const dictionary& decompDict, + const word& regionName +) +: + decompositionDict_(decompDict), + decompositionRegionDict_ + ( + optionalRegionDict(decompositionDict_, regionName) + ), + nDomains_(nDomains(decompDict, regionName)) +{ + readConstraints(); +} + // * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // Foam::autoPtr Foam::decompositionMethod::New ( - const dictionary& decompositionDict + const dictionary& decompDict ) { - const word methodType(decompositionDict.lookup("method")); - - Info<< "Selecting decompositionMethod " << methodType << endl; + const word methodType(decompDict.lookup("method")); auto cstrIter = dictionaryConstructorTablePtr_->cfind(methodType); @@ -190,7 +365,58 @@ Foam::autoPtr Foam::decompositionMethod::New << exit(FatalError); } - return autoPtr(cstrIter()(decompositionDict)); + // verbose + { + Info<< "Selecting decompositionMethod " << methodType + << " [" << (nDomains(decompDict)) << "]" << endl; + } + + return autoPtr(cstrIter()(decompDict)); +} + + +Foam::autoPtr Foam::decompositionMethod::New +( + const dictionary& decompDict, + const word& regionName + +) +{ + const dictionary& regionDict(optionalRegionDict(decompDict, regionName)); + + if (regionDict.empty()) + { + // No region-specific information - just forward to normal routine + return decompositionMethod::New(decompDict); + } + + word methodType(decompDict.lookup("method")); + regionDict.readIfPresent("method", methodType); + + auto cstrIter = dictionaryRegionConstructorTablePtr_->cfind(methodType); + + if (!cstrIter.found()) + { + WarningInFunction + << nl + << "Unknown region decompositionMethod " + << methodType << nl << nl + << "Valid decompositionMethods : " << endl + << dictionaryRegionConstructorTablePtr_->sortedToc() << nl + << "Reverting to non-region version" << nl + << endl; + + return decompositionMethod::New(decompDict); + } + + // verbose + { + Info<< "Selecting decompositionMethod " << methodType + << " [" << (nDomains(decompDict, regionName)) << "] (region " + << regionName << ")" << endl; + } + + return autoPtr(cstrIter()(decompDict, regionName)); } @@ -341,7 +567,7 @@ void Foam::decompositionMethod::calcCellCells // Number of faces per coarse cell labelList nFacesPerCell(nLocalCoarse, 0); - for (label facei = 0; facei < mesh.nInternalFaces(); facei++) + for (label facei = 0; facei < mesh.nInternalFaces(); ++facei) { const label own = agglom[faceOwner[facei]]; const label nei = agglom[faceNeighbour[facei]]; @@ -391,7 +617,7 @@ void Foam::decompositionMethod::calcCellCells const labelList& offsets = cellCells.offsets(); // For internal faces is just offsetted owner and neighbour - for (label facei = 0; facei < mesh.nInternalFaces(); facei++) + for (label facei = 0; facei < mesh.nInternalFaces(); ++facei) { const label own = agglom[faceOwner[facei]]; const label nei = agglom[faceNeighbour[facei]]; @@ -453,7 +679,7 @@ void Foam::decompositionMethod::calcCellCells const label endIndex = cellCells.offsets()[celli+1]; - for (label i = startIndex; i < endIndex; i++) + for (label i = startIndex; i < endIndex; ++i) { if (nbrCells.insert(cellCells.m()[i])) { @@ -547,10 +773,10 @@ void Foam::decompositionMethod::calcCellCells // Number of faces per coarse cell labelList nFacesPerCell(nLocalCoarse, 0); - for (label faceI = 0; faceI < mesh.nInternalFaces(); faceI++) + for (label facei = 0; facei < mesh.nInternalFaces(); ++facei) { - const label own = agglom[faceOwner[faceI]]; - const label nei = agglom[faceNeighbour[faceI]]; + const label own = agglom[faceOwner[facei]]; + const label nei = agglom[faceNeighbour[facei]]; nFacesPerCell[own]++; nFacesPerCell[nei]++; @@ -599,18 +825,18 @@ void Foam::decompositionMethod::calcCellCells const labelList& offsets = cellCells.offsets(); // For internal faces is just offsetted owner and neighbour - for (label faceI = 0; faceI < mesh.nInternalFaces(); faceI++) + for (label facei = 0; facei < mesh.nInternalFaces(); ++facei) { - const label own = agglom[faceOwner[faceI]]; - const label nei = agglom[faceNeighbour[faceI]]; + const label own = agglom[faceOwner[facei]]; + const label nei = agglom[faceNeighbour[facei]]; const label ownIndex = offsets[own] + nFacesPerCell[own]++; const label neiIndex = offsets[nei] + nFacesPerCell[nei]++; m[ownIndex] = globalAgglom.toGlobal(nei); - w[ownIndex] = mag(mesh.faceAreas()[faceI]); + w[ownIndex] = mag(mesh.faceAreas()[facei]); m[neiIndex] = globalAgglom.toGlobal(own); - w[ownIndex] = mag(mesh.faceAreas()[faceI]); + w[ownIndex] = mag(mesh.faceAreas()[facei]); } // For boundary faces is offsetted coupled neighbour @@ -661,14 +887,14 @@ void Foam::decompositionMethod::calcCellCells label startIndex = cellCells.offsets()[0]; - forAll(cellCells, cellI) + forAll(cellCells, celli) { nbrCells.clear(); - nbrCells.insert(globalAgglom.toGlobal(cellI)); + nbrCells.insert(globalAgglom.toGlobal(celli)); - const label endIndex = cellCells.offsets()[cellI+1]; + const label endIndex = cellCells.offsets()[celli+1]; - for (label i = startIndex; i < endIndex; i++) + for (label i = startIndex; i < endIndex; ++i) { if (nbrCells.insert(cellCells.m()[i])) { @@ -678,8 +904,8 @@ void Foam::decompositionMethod::calcCellCells } } startIndex = endIndex; - cellCells.offsets()[cellI+1] = newIndex; - cellCellWeights.offsets()[cellI+1] = newIndex; + cellCells.offsets()[celli+1] = newIndex; + cellCellWeights.offsets()[celli+1] = newIndex; } cellCells.m().setSize(newIndex); @@ -753,7 +979,7 @@ void Foam::decompositionMethod::calcCellCells // labelList nFacesPerCell(nLocalCoarse, 0); // // // 1. Internal faces -// for (label facei = 0; facei < mesh.nInternalFaces(); facei++) +// for (label facei = 0; facei < mesh.nInternalFaces(); ++facei) // { // if (!blockedFace[facei]) // { @@ -880,7 +1106,7 @@ void Foam::decompositionMethod::calcCellCells // const labelList& offsets = cellCells.offsets(); // // // 1. For internal faces is just offsetted owner and neighbour -// for (label facei = 0; facei < mesh.nInternalFaces(); facei++) +// for (label facei = 0; facei < mesh.nInternalFaces(); ++facei) // { // if (!blockedFace[facei]) // { @@ -1032,7 +1258,7 @@ void Foam::decompositionMethod::calcCellCells // // label endIndex = cellCells.offsets()[celli+1]; // -// for (label i = startIndex; i < endIndex; i++) +// for (label i = startIndex; i < endIndex; ++i) // { // if (nbrCells.insert(cellCells.m()[i])) // { @@ -1180,11 +1406,11 @@ Foam::labelList Foam::decompositionMethod::decompose forAll(localRegion, celli) { - label regionI = localRegion[celli]; + const label regioni = localRegion[celli]; - if (regionCentres[regionI] == point::max) + if (regionCentres[regioni] == point::max) { - regionCentres[regionI] = mesh.cellCentres()[celli]; + regionCentres[regioni] = mesh.cellCentres()[celli]; } } @@ -1197,18 +1423,18 @@ Foam::labelList Foam::decompositionMethod::decompose { forAll(localRegion, celli) { - label regionI = localRegion[celli]; + const label regioni = localRegion[celli]; - regionWeights[regionI] += cellWeights[celli]; + regionWeights[regioni] += cellWeights[celli]; } } else { forAll(localRegion, celli) { - label regionI = localRegion[celli]; + const label regioni = localRegion[celli]; - regionWeights[regionI] += 1.0; + regionWeights[regioni] += 1.0; } } @@ -1224,11 +1450,11 @@ Foam::labelList Foam::decompositionMethod::decompose // Implement the explicitConnections since above decompose // does not know about them - forAll(explicitConnections, i) + forAll(explicitConnections, connectioni) { - const labelPair& baffle = explicitConnections[i]; - label f0 = baffle.first(); - label f1 = baffle.second(); + const labelPair& baffle = explicitConnections[connectioni]; + const label f0 = baffle.first(); + const label f1 = baffle.second(); if (!blockedFace[f0] && !blockedFace[f1]) { @@ -1292,7 +1518,7 @@ Foam::labelList Foam::decompositionMethod::decompose { if (!blockedFace[facei]) { - label own = mesh.faceOwner()[facei]; + const label own = mesh.faceOwner()[facei]; seedFaces[nUnblocked] = facei; seedData[nUnblocked] = minData(finalDecomp[own]); nUnblocked++; @@ -1361,10 +1587,8 @@ Foam::labelList Foam::decompositionMethod::decompose forAll(f, fp) { const labelList& pFaces = mesh.pointFaces()[f[fp]]; - forAll(pFaces, i) + for (const label facei : pFaces) { - label facei = pFaces[i]; - finalDecomp[mesh.faceOwner()[facei]] = proci; if (mesh.isInternalFace(facei)) { @@ -1433,9 +1657,9 @@ void Foam::decompositionMethod::setConstraints specifiedProcessorFaces.clear(); explicitConnections.clear(); - forAll(constraints_, constraintI) + forAll(constraints_, constrainti) { - constraints_[constraintI].add + constraints_[constrainti].add ( mesh, blockedFace, @@ -1457,9 +1681,9 @@ void Foam::decompositionMethod::applyConstraints labelList& decomposition ) { - forAll(constraints_, constraintI) + forAll(constraints_, constrainti) { - constraints_[constraintI].apply + constraints_[constrainti].apply ( mesh, blockedFace, diff --git a/src/parallel/decompose/decompositionMethods/decompositionMethod/decompositionMethod.H b/src/parallel/decompose/decompositionMethods/decompositionMethod/decompositionMethod.H index 75a797e1ae..96ad3afc98 100644 --- a/src/parallel/decompose/decompositionMethods/decompositionMethod/decompositionMethod.H +++ b/src/parallel/decompose/decompositionMethods/decompositionMethod/decompositionMethod.H @@ -25,7 +25,7 @@ Class Foam::decompositionMethod Description - Abstract base class for decomposition + Abstract base class for domain decomposition SourceFiles decompositionMethod.C @@ -43,29 +43,86 @@ namespace Foam { /*---------------------------------------------------------------------------*\ - Class decompositionMethod Declaration + Class decompositionMethod Declaration \*---------------------------------------------------------------------------*/ class decompositionMethod { + // Private Member Functions + + //- Set PtrList of constraints by reading decompositionDict_. + void readConstraints(); + + //- Disallow default bitwise copy construct and assignment + decompositionMethod(const decompositionMethod&) = delete; + void operator=(const decompositionMethod&) = delete; + protected: + //- Selection type when handling the coefficients dictionary. + // To be used as a bit-mask for findCoeffsDict + enum selectionType + { + DEFAULT = 0, //!< Default request + EXACT = 1, //!< No fallback to "coeffs" if main name not found + MANDATORY = 2, //!< Fatal if dictionary could not be found + NULL_DICT = 4, //!< On failure, return dictionary::null instead + //!< of the top-level enclosing dictionary. + }; + + // Protected data + //- Top-level decomposition dictionary (eg, decomposeParDict) const dictionary& decompositionDict_; - label nProcessors_; + + //- Region-specific decomposition dictionary information + const dictionary& decompositionRegionDict_; + + //- Number of domains for the decomposition + label nDomains_; //- Optional constraints PtrList constraints_; -private: - // Private Member Functions + // Protected Member Functions - //- Disallow default bitwise copy construct and assignment - decompositionMethod(const decompositionMethod&); - void operator=(const decompositionMethod&); + //- Locate coeffsName dictionary or the fallback "coeffs" dictionary + //- within an enclosing dictionary. + // + // \param select choose to include "coeffs" in the search, make + // failure a FatalError, return dictionary::null instead on + // failure. + // + // \return the coefficients dictionary found. If nothing was found, + // return the enclosing dictionary or + // dictionary::null (depending on the select parameter). + static const dictionary& findCoeffsDict + ( + const dictionary& dict, + const word& coeffsName, + int select = selectionType::DEFAULT + ); + + + //- Locate coeffsName dictionary or the fallback "coeffs" dictionary. + // Searches both the region-specific decomposition dictionary + // and the top-level decomposition dictionary. + // + // \param select choose to include "coeffs" in the search, make + // failure a FatalError, return dictionary::null instead on + // failure. + // + // \return the coefficients dictionary found. If nothing was found, + // return the top-level (non-region) dictionary or + // dictionary::null (depending on the select parameter). + const dictionary& findCoeffsDict + ( + const word& coeffsName, + int select = selectionType::DEFAULT + ) const; public: @@ -82,9 +139,43 @@ public: decompositionMethod, dictionary, ( - const dictionary& decompositionDict + const dictionary& decompDict ), - (decompositionDict) + (decompDict) + ); + + declareRunTimeSelectionTable + ( + autoPtr, + decompositionMethod, + dictionaryRegion, + ( + const dictionary& decompDict, + const word& regionName + ), + (decompDict, regionName) + ); + + + // Static Methods + + //- Return the \c numberOfSubdomains entry from the dictionary + static label nDomains(const dictionary& decompDict); + + //- Return the \c numberOfSubdomains from a region within the + // "regions" sub-dictionary + static label nDomains + ( + const dictionary& decompDict, + const word& regionName + ); + + //- Return an optional region dictionary from "regions" sub-dictionary + // or dictionary::null on failure. + static const dictionary& optionalRegionDict + ( + const dictionary& decompDict, + const word& regionName ); @@ -93,14 +184,29 @@ public: //- Return a reference to the selected decomposition method static autoPtr New ( - const dictionary& decompositionDict + const dictionary& decompDict + ); + + //- Return a reference to the selected decomposition method, + //- with region specification + static autoPtr New + ( + const dictionary& decompDict, + const word& regionName ); // Constructors //- Construct given the decomposition dictionary - decompositionMethod(const dictionary& decompositionDict); + decompositionMethod(const dictionary& decompDict); + + //- Construct given the decomposition dictionary for specific region + decompositionMethod + ( + const dictionary& decompDict, + const word& regionName + ); //- Destructor @@ -110,9 +216,10 @@ public: // Member Functions - label nDomains() const + //- Number of domains + inline label nDomains() const { - return nProcessors_; + return nDomains_; } //- Is method parallel aware (i.e. does it synchronize domains across diff --git a/src/parallel/decompose/decompositionMethods/geomDecomp/geomDecomp.C b/src/parallel/decompose/decompositionMethods/geomDecomp/geomDecomp.C index 6f741b0959..1c2df26498 100644 --- a/src/parallel/decompose/decompositionMethods/geomDecomp/geomDecomp.C +++ b/src/parallel/decompose/decompositionMethods/geomDecomp/geomDecomp.C @@ -25,36 +25,29 @@ License #include "geomDecomp.H" -// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * // +// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * // -Foam::geomDecomp::geomDecomp -( - const dictionary& decompositionDict, - const word& derivedType -) -: - decompositionMethod(decompositionDict), - geomDecomDict_(decompositionDict.optionalSubDict(derivedType + "Coeffs")), - n_(geomDecomDict_.lookup("n")), - delta_(readScalar(geomDecomDict_.lookup("delta"))), - rotDelta_(I) +void Foam::geomDecomp::readCoeffs() { - // check that the case makes sense : + coeffsDict_.readIfPresent("delta", delta_); - if (nProcessors_ != n_.x()*n_.y()*n_.z()) + coeffsDict_.lookup("n") >> n_; + + // Verify that the input makes sense + if (nDomains_ != n_.x()*n_.y()*n_.z()) { FatalErrorInFunction - << "Wrong number of processor divisions in geomDecomp:" << nl - << "Number of domains : " << nProcessors_ << nl + << "Wrong number of domain divisions in geomDecomp:" << nl + << "Number of domains : " << nDomains_ << nl << "Wanted decomposition : " << n_ << exit(FatalError); } - scalar d = 1 - 0.5*delta_*delta_; - scalar d2 = sqr(d); + const scalar d = 1 - 0.5*delta_*delta_; + const scalar d2 = sqr(d); - scalar a = delta_; - scalar a2 = sqr(a); + const scalar a = delta_; + const scalar a2 = sqr(a); rotDelta_ = tensor ( @@ -65,4 +58,41 @@ Foam::geomDecomp::geomDecomp } +// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * // + +Foam::geomDecomp::geomDecomp +( + const word& derivedType, + const dictionary& decompDict, + int select +) +: + decompositionMethod(decompDict), + coeffsDict_(findCoeffsDict(derivedType + "Coeffs", select)), + n_(1,1,1), + delta_(0.001), + rotDelta_(I) +{ + readCoeffs(); +} + + +Foam::geomDecomp::geomDecomp +( + const word& derivedType, + const dictionary& decompDict, + const word& regionName, + int select +) +: + decompositionMethod(decompDict, regionName), + coeffsDict_(findCoeffsDict(derivedType + "Coeffs", select)), + n_(1,1,1), + delta_(0.001), + rotDelta_(I) +{ + readCoeffs(); +} + + // ************************************************************************* // diff --git a/src/parallel/decompose/decompositionMethods/geomDecomp/geomDecomp.H b/src/parallel/decompose/decompositionMethods/geomDecomp/geomDecomp.H index e1538356ee..7684a2ef4e 100644 --- a/src/parallel/decompose/decompositionMethods/geomDecomp/geomDecomp.H +++ b/src/parallel/decompose/decompositionMethods/geomDecomp/geomDecomp.H @@ -25,7 +25,14 @@ Class Foam::geomDecomp Description - Geometrical domain decomposition + Base for geometrical domain decomposition methods + + Base coefficients: + \table + Property | Description | Required | Default + n | (nx ny nz) | yes + delta | delta for rotation matrix | no | 0.001 + \endtable SourceFiles geomDecomp.C @@ -42,36 +49,56 @@ namespace Foam { /*---------------------------------------------------------------------------*\ - Class geomDecomp Declaration + Class geomDecomp Declaration \*---------------------------------------------------------------------------*/ class geomDecomp : public decompositionMethod { + // Private Member Functions + + //- Read input values and initialize the rotDelta_ + void readCoeffs(); + protected: // Protected data - const dictionary& geomDecomDict_; + //- Coefficients for all derived methods + const dictionary& coeffsDict_; Vector