ENH: parallel: overhaul of parallel mapping

- redistributePar to have almost (complete) functionality of decomposePar+reconstructPar
- low-level distributed Field mapping
- support for mapping surfaceFields (including flipping faces)
- support for decomposing/reconstructing refinement data
This commit is contained in:
mattijs
2015-11-17 15:05:05 +00:00
parent 09dec5263b
commit 916dcb8685
205 changed files with 21665 additions and 4196 deletions

View File

@ -0,0 +1,41 @@
/*--------------------------------*- C++ -*----------------------------------*\
| ========= | |
| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox |
| \\ / O peration | Version: dev |
| \\ / A nd | Web: www.OpenFOAM.org |
| \\/ M anipulation | |
\*---------------------------------------------------------------------------*/
FoamFile
{
version 2.0;
format ascii;
class volVectorField;
object U;
}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
dimensions [0 1 -1 0 0 0 0];
internalField uniform (0 0 0);
boundaryField
{
movingWall
{
type fixedValue;
value uniform (1 0 0);
}
fixedWalls
{
type fixedValue;
value uniform (0 0 0);
}
frontAndBack
{
type empty;
}
}
// ************************************************************************* //

View File

@ -0,0 +1,39 @@
/*--------------------------------*- C++ -*----------------------------------*\
| ========= | |
| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox |
| \\ / O peration | Version: dev |
| \\ / A nd | Web: www.OpenFOAM.org |
| \\/ M anipulation | |
\*---------------------------------------------------------------------------*/
FoamFile
{
version 2.0;
format ascii;
class volScalarField;
object p;
}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
dimensions [0 2 -2 0 0 0 0];
internalField uniform 0;
boundaryField
{
movingWall
{
type zeroGradient;
}
fixedWalls
{
type zeroGradient;
}
frontAndBack
{
type empty;
}
}
// ************************************************************************* //

View File

@ -0,0 +1,12 @@
#!/bin/sh
# Source tutorial clean functions
. $WM_PROJECT_DIR/bin/tools/CleanFunctions
# Restore default dictionaries
cp system/decomposeParDict-2 system/decomposeParDict
cp system/controlDict-startTime system/controlDict
cleanCase
rm -f constant/polyMesh/boundary
rm -f constant/polyMesh/procAddressing

View File

@ -0,0 +1,35 @@
#!/bin/sh
cd ${0%/*} || exit 1 # Run from this directory
# Source tutorial run functions
. $WM_PROJECT_DIR/bin/tools/RunFunctions
runApplication blockMesh
#cp system/decomposeParDict-2 system/decomposeParDict
#runApplication decomposePar
#- redistributePar to do decomposition
runParallel redistributePar 2 -decompose
#- bit of renumbering and running
runParallel -l log.renumberMesh-CuthillMcKee renumberMesh 2 -overwrite
runParallel -l log.icoFoam-CuthillMcKee icoFoam 2
#- bit of bad renumbering and running
runParallel -l log.renumberMesh-parallel renumberMesh 2 -overwrite -dict system/renumberMeshDict-random
runParallel -l log.icoFoam-random icoFoam 2
#- pick up last result
cp system/controlDict-latestTime system/controlDict
#- redistribute to 5 processors
runParallel -l log.redistributePar-5 redistributePar 5 -decomposeParDict system/decomposeParDict-5 -cellDist
#- run a bit more
runParallel -l log.icoFoam-5 icoFoam 5 -decomposeParDict system/decomposeParDict-5
#- reconstruct mesh and results
runParallel -l log.redistributePar-1 redistributePar 5 -reconstruct -decomposeParDict system/decomposeParDict
# ----------------------------------------------------------------- end-of-file

View File

@ -0,0 +1,75 @@
/*--------------------------------*- C++ -*----------------------------------*\
| ========= | |
| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox |
| \\ / O peration | Version: dev |
| \\ / A nd | Web: www.OpenFOAM.org |
| \\/ M anipulation | |
\*---------------------------------------------------------------------------*/
FoamFile
{
version 2.0;
format ascii;
class dictionary;
object blockMeshDict;
}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
convertToMeters 0.1;
vertices
(
(0 0 0)
(1 0 0)
(1 1 0)
(0 1 0)
(0 0 0.1)
(1 0 0.1)
(1 1 0.1)
(0 1 0.1)
);
blocks
(
hex (0 1 2 3 4 5 6 7) (20 20 1) simpleGrading (1 1 1)
);
edges
(
);
boundary
(
movingWall
{
type wall;
faces
(
(3 7 6 2)
);
}
fixedWalls
{
type wall;
faces
(
(0 4 7 3)
(2 6 5 1)
(1 5 4 0)
);
}
frontAndBack
{
type empty;
faces
(
(0 3 2 1)
(4 5 6 7)
);
}
);
mergePatchPairs
(
);
// ************************************************************************* //

View File

@ -0,0 +1,21 @@
/*--------------------------------*- C++ -*----------------------------------*\
| ========= | |
| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox |
| \\ / O peration | Version: dev |
| \\ / A nd | Web: www.OpenFOAM.org |
| \\/ M anipulation | |
\*---------------------------------------------------------------------------*/
FoamFile
{
version 2.0;
format ascii;
class dictionary;
location "constant";
object transportProperties;
}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
nu nu [ 0 2 -1 0 0 0 0 ] 0.01;
// ************************************************************************* //

View File

@ -0,0 +1,49 @@
/*--------------------------------*- C++ -*----------------------------------*\
| ========= | |
| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox |
| \\ / O peration | Version: dev |
| \\ / A nd | Web: www.OpenFOAM.org |
| \\/ M anipulation | |
\*---------------------------------------------------------------------------*/
FoamFile
{
version 2.0;
format ascii;
class dictionary;
location "system";
object controlDict;
}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
application icoFoam;
startFrom startTime;
startTime 0;
stopAt endTime;
endTime 0.5;
deltaT 0.005;
writeControl timeStep;
writeInterval 20;
purgeWrite 0;
writeFormat ascii;
writePrecision 6;
writeCompression off;
timeFormat general;
timePrecision 6;
runTimeModifiable true;
// ************************************************************************* //

View File

@ -0,0 +1,49 @@
/*--------------------------------*- C++ -*----------------------------------*\
| ========= | |
| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox |
| \\ / O peration | Version: dev |
| \\ / A nd | Web: www.OpenFOAM.org |
| \\/ M anipulation | |
\*---------------------------------------------------------------------------*/
FoamFile
{
version 2.0;
format ascii;
class dictionary;
location "system";
object controlDict;
}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
application icoFoam;
startFrom latestTime;
startTime 0;
stopAt endTime;
endTime 1.0;
deltaT 0.005;
writeControl timeStep;
writeInterval 20;
purgeWrite 0;
writeFormat ascii;
writePrecision 6;
writeCompression off;
timeFormat general;
timePrecision 6;
runTimeModifiable true;
// ************************************************************************* //

View File

@ -0,0 +1,49 @@
/*--------------------------------*- C++ -*----------------------------------*\
| ========= | |
| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox |
| \\ / O peration | Version: dev |
| \\ / A nd | Web: www.OpenFOAM.org |
| \\/ M anipulation | |
\*---------------------------------------------------------------------------*/
FoamFile
{
version 2.0;
format ascii;
class dictionary;
location "system";
object controlDict;
}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
application icoFoam;
startFrom startTime;
startTime 0;
stopAt endTime;
endTime 0.5;
deltaT 0.005;
writeControl timeStep;
writeInterval 20;
purgeWrite 0;
writeFormat ascii;
writePrecision 6;
writeCompression off;
timeFormat general;
timePrecision 6;
runTimeModifiable true;
// ************************************************************************* //

View File

@ -0,0 +1,143 @@
/*--------------------------------*- C++ -*----------------------------------*\
| ========= | |
| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox |
| \\ / O peration | Version: dev |
| \\ / A nd | Web: www.OpenFOAM.org |
| \\/ M anipulation | |
\*---------------------------------------------------------------------------*/
FoamFile
{
version 2.0;
format ascii;
class dictionary;
note "mesh decomposition control dictionary";
object decomposeParDict;
}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
numberOfSubdomains 2;
//- Keep owner and neighbour on same processor for faces in zones:
// preserveFaceZones (heater solid1 solid3);
//- Keep owner and neighbour on same processor for faces in patches:
// (makes sense only for cyclic patches)
//preservePatches (cyclic_half0 cyclic_half1);
//- Keep all of faceSet on a single processor. This puts all cells
// connected with a point, edge or face on the same processor.
// (just having face connected cells might not guarantee a balanced
// decomposition)
// The processor can be -1 (the decompositionMethod chooses the processor
// for a good load balance) or explicitly provided (upsets balance).
//singleProcessorFaceSets ((f0 -1));
//- Keep owner and neighbour of baffles on same processor (i.e. keep it
// detectable as a baffle). Baffles are two boundary face sharing the
// same points.
//preserveBaffles true;
//- Use the volScalarField named here as a weight for each cell in the
// decomposition. For example, use a particle population field to decompose
// for a balanced number of particles in a lagrangian simulation.
// weightField dsmcRhoNMean;
method scotch;
//method hierarchical;
// method simple;
// method metis;
// method manual;
// method multiLevel;
// method structured; // does 2D decomposition of structured mesh
multiLevelCoeffs
{
// Decomposition methods to apply in turn. This is like hierarchical but
// fully general - every method can be used at every level.
level0
{
numberOfSubdomains 64;
//method simple;
//simpleCoeffs
//{
// n (2 1 1);
// delta 0.001;
//}
method scotch;
}
level1
{
numberOfSubdomains 4;
method scotch;
}
}
// Desired output
simpleCoeffs
{
n (2 1 1);
delta 0.001;
}
hierarchicalCoeffs
{
n (1 2 1);
delta 0.001;
order xyz;
}
metisCoeffs
{
/*
processorWeights
(
1
1
1
1
);
*/
}
scotchCoeffs
{
//processorWeights
//(
// 1
// 1
// 1
// 1
//);
//writeGraph true;
//strategy "b";
}
manualCoeffs
{
dataFile "decompositionData";
}
structuredCoeffs
{
// Patches to do 2D decomposition on. Structured mesh only; cells have
// to be in 'columns' on top of patches.
patches (movingWall);
// Method to use on the 2D subset
method scotch;
}
//// Is the case distributed? Note: command-line argument -roots takes
//// precedence
//distributed yes;
//// Per slave (so nProcs-1 entries) the directory above the case.
//roots
//(
// "/tmp"
// "/tmp"
//);
// ************************************************************************* //

View File

@ -0,0 +1,143 @@
/*--------------------------------*- C++ -*----------------------------------*\
| ========= | |
| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox |
| \\ / O peration | Version: dev |
| \\ / A nd | Web: www.OpenFOAM.org |
| \\/ M anipulation | |
\*---------------------------------------------------------------------------*/
FoamFile
{
version 2.0;
format ascii;
class dictionary;
note "mesh decomposition control dictionary";
object decomposeParDict;
}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
numberOfSubdomains 2;
//- Keep owner and neighbour on same processor for faces in zones:
// preserveFaceZones (heater solid1 solid3);
//- Keep owner and neighbour on same processor for faces in patches:
// (makes sense only for cyclic patches)
//preservePatches (cyclic_half0 cyclic_half1);
//- Keep all of faceSet on a single processor. This puts all cells
// connected with a point, edge or face on the same processor.
// (just having face connected cells might not guarantee a balanced
// decomposition)
// The processor can be -1 (the decompositionMethod chooses the processor
// for a good load balance) or explicitly provided (upsets balance).
//singleProcessorFaceSets ((f0 -1));
//- Keep owner and neighbour of baffles on same processor (i.e. keep it
// detectable as a baffle). Baffles are two boundary face sharing the
// same points.
//preserveBaffles true;
//- Use the volScalarField named here as a weight for each cell in the
// decomposition. For example, use a particle population field to decompose
// for a balanced number of particles in a lagrangian simulation.
// weightField dsmcRhoNMean;
method scotch;
//method hierarchical;
// method simple;
// method metis;
// method manual;
// method multiLevel;
// method structured; // does 2D decomposition of structured mesh
multiLevelCoeffs
{
// Decomposition methods to apply in turn. This is like hierarchical but
// fully general - every method can be used at every level.
level0
{
numberOfSubdomains 64;
//method simple;
//simpleCoeffs
//{
// n (2 1 1);
// delta 0.001;
//}
method scotch;
}
level1
{
numberOfSubdomains 4;
method scotch;
}
}
// Desired output
simpleCoeffs
{
n (2 1 1);
delta 0.001;
}
hierarchicalCoeffs
{
n (1 2 1);
delta 0.001;
order xyz;
}
metisCoeffs
{
/*
processorWeights
(
1
1
1
1
);
*/
}
scotchCoeffs
{
//processorWeights
//(
// 1
// 1
// 1
// 1
//);
//writeGraph true;
//strategy "b";
}
manualCoeffs
{
dataFile "decompositionData";
}
structuredCoeffs
{
// Patches to do 2D decomposition on. Structured mesh only; cells have
// to be in 'columns' on top of patches.
patches (movingWall);
// Method to use on the 2D subset
method scotch;
}
//// Is the case distributed? Note: command-line argument -roots takes
//// precedence
//distributed yes;
//// Per slave (so nProcs-1 entries) the directory above the case.
//roots
//(
// "/tmp"
// "/tmp"
//);
// ************************************************************************* //

View File

@ -0,0 +1,143 @@
/*--------------------------------*- C++ -*----------------------------------*\
| ========= | |
| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox |
| \\ / O peration | Version: dev |
| \\ / A nd | Web: www.OpenFOAM.org |
| \\/ M anipulation | |
\*---------------------------------------------------------------------------*/
FoamFile
{
version 2.0;
format ascii;
class dictionary;
note "mesh decomposition control dictionary";
object decomposeParDict;
}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
numberOfSubdomains 5;
//- Keep owner and neighbour on same processor for faces in zones:
// preserveFaceZones (heater solid1 solid3);
//- Keep owner and neighbour on same processor for faces in patches:
// (makes sense only for cyclic patches)
//preservePatches (cyclic_half0 cyclic_half1);
//- Keep all of faceSet on a single processor. This puts all cells
// connected with a point, edge or face on the same processor.
// (just having face connected cells might not guarantee a balanced
// decomposition)
// The processor can be -1 (the decompositionMethod chooses the processor
// for a good load balance) or explicitly provided (upsets balance).
//singleProcessorFaceSets ((f0 -1));
//- Keep owner and neighbour of baffles on same processor (i.e. keep it
// detectable as a baffle). Baffles are two boundary face sharing the
// same points.
//preserveBaffles true;
//- Use the volScalarField named here as a weight for each cell in the
// decomposition. For example, use a particle population field to decompose
// for a balanced number of particles in a lagrangian simulation.
// weightField dsmcRhoNMean;
method scotch;
//method hierarchical;
// method simple;
// method metis;
// method manual;
// method multiLevel;
// method structured; // does 2D decomposition of structured mesh
multiLevelCoeffs
{
// Decomposition methods to apply in turn. This is like hierarchical but
// fully general - every method can be used at every level.
level0
{
numberOfSubdomains 64;
//method simple;
//simpleCoeffs
//{
// n (2 1 1);
// delta 0.001;
//}
method scotch;
}
level1
{
numberOfSubdomains 4;
method scotch;
}
}
// Desired output
simpleCoeffs
{
n (2 1 1);
delta 0.001;
}
hierarchicalCoeffs
{
n (1 2 1);
delta 0.001;
order xyz;
}
metisCoeffs
{
/*
processorWeights
(
1
1
1
1
);
*/
}
scotchCoeffs
{
//processorWeights
//(
// 1
// 1
// 1
// 1
//);
//writeGraph true;
//strategy "b";
}
manualCoeffs
{
dataFile "decompositionData";
}
structuredCoeffs
{
// Patches to do 2D decomposition on. Structured mesh only; cells have
// to be in 'columns' on top of patches.
patches (movingWall);
// Method to use on the 2D subset
method scotch;
}
//// Is the case distributed? Note: command-line argument -roots takes
//// precedence
//distributed yes;
//// Per slave (so nProcs-1 entries) the directory above the case.
//roots
//(
// "/tmp"
// "/tmp"
//);
// ************************************************************************* //

View File

@ -0,0 +1,57 @@
/*--------------------------------*- C++ -*----------------------------------*\
| ========= | |
| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox |
| \\ / O peration | Version: dev |
| \\ / A nd | Web: www.OpenFOAM.org |
| \\/ M anipulation | |
\*---------------------------------------------------------------------------*/
FoamFile
{
version 2.0;
format ascii;
class dictionary;
location "system";
object fvSchemes;
}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
ddtSchemes
{
default Euler;
}
gradSchemes
{
default Gauss linear;
grad(p) Gauss linear;
}
divSchemes
{
default none;
div(phi,U) Gauss linear;
}
laplacianSchemes
{
default Gauss linear orthogonal;
}
interpolationSchemes
{
default linear;
}
snGradSchemes
{
default orthogonal;
}
fluxRequired
{
default no;
p ;
}
// ************************************************************************* //

View File

@ -0,0 +1,46 @@
/*--------------------------------*- C++ -*----------------------------------*\
| ========= | |
| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox |
| \\ / O peration | Version: dev |
| \\ / A nd | Web: www.OpenFOAM.org |
| \\/ M anipulation | |
\*---------------------------------------------------------------------------*/
FoamFile
{
version 2.0;
format ascii;
class dictionary;
location "system";
object fvSolution;
}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
solvers
{
p
{
solver PCG;
preconditioner DIC;
tolerance 1e-06;
relTol 0;
}
U
{
solver smoothSolver;
smoother symGaussSeidel;
tolerance 1e-05;
relTol 0;
}
}
PISO
{
nCorrectors 2;
nNonOrthogonalCorrectors 0;
pRefCell 0;
pRefValue 0;
}
// ************************************************************************* //

View File

@ -0,0 +1,112 @@
/*--------------------------------*- C++ -*----------------------------------*\
| ========= | |
| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox |
| \\ / O peration | Version: dev |
| \\ / A nd | Web: www.OpenFOAM.org |
| \\/ M anipulation | |
\*---------------------------------------------------------------------------*/
FoamFile
{
version 2.0;
format ascii;
class dictionary;
note "mesh renumbering dictionary";
object renumberMeshDict;
}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
// Write maps from renumbered back to original mesh
writeMaps true;
// Optional entry: sort cells on coupled boundaries to last for use with
// e.g. nonBlockingGaussSeidel.
sortCoupledFaceCells false;
// Optional entry: renumber on a block-by-block basis. It uses a
// blockCoeffs dictionary to construct a decompositionMethod to do
// a block subdivision) and then applies the renumberMethod to each
// block in turn. This can be used in large cases to keep the blocks
// fitting in cache with all the the cache misses bunched at the end.
// This number is the approximate size of the blocks - this gets converted
// to a number of blocks that is the input to the decomposition method.
//blockSize 1000;
// Optional entry: sort points into internal and boundary points
//orderPoints false;
// Optional: suppress renumbering cellSets,faceSets,pointSets
//renumberSets false;
//method CuthillMcKee;
//method Sloan;
//method manual;
method random;
//method structured;
//method spring;
//method zoltan; // only if compiled with zoltan support
//CuthillMcKeeCoeffs
//{
// // Reverse CuthillMcKee (RCM) or plain
// reverse true;
//}
manualCoeffs
{
// In system directory: new-to-original (i.e. order) labelIOList
dataFile "cellMap";
}
// For extruded (i.e. structured in one direction) meshes
structuredCoeffs
{
// Patches that mesh was extruded from. These determine the starting
// layer of cells
patches (movingWall);
// Method to renumber the starting layer of cells
method random;
// Renumber in columns (depthFirst) or in layers
depthFirst true;
// Reverse ordering
reverse false;
}
springCoeffs
{
// Maximum jump of cell indices. Is fraction of number of cells
maxCo 0.01;
// Limit the amount of movement; the fraction maxCo gets decreased
// with every iteration
freezeFraction 0.999;
// Maximum number of iterations
maxIter 1000;
}
blockCoeffs
{
method scotch;
//method hierarchical;
//hierarchicalCoeffs
//{
// n (1 2 1);
// delta 0.001;
// order xyz;
//}
}
zoltanCoeffs
{
ORDER_METHOD LOCAL_HSFC;
}
// ************************************************************************* //