zhyang-dev/openfoam
1
0
Fork 0
mirror of https://develop.openfoam.com/Development/openfoam.git synced 2025-11-28 03:28:01 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

ENH: parallel: overhaul of parallel mapping

Browse Source 
- redistributePar to have almost (complete) functionality of decomposePar+reconstructPar
- low-level distributed Field mapping
- support for mapping surfaceFields (including flipping faces)
- support for decomposing/reconstructing refinement data
This commit is contained in:
mattijs
2015-11-17 15:05:05 +00:00
parent 09dec5263b
commit 916dcb8685
205 changed files with 21665 additions and 4196 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
applications/utilities/parallelProcessing/decomposePar/Make/options
View File

@ -2,6 +2,7 @@ EXE_INC = \
-I$(LIB_SRC)/parallel/decompose/decompose/lnInclude \
-I$(LIB_SRC)/parallel/decompose/decompositionMethods/lnInclude \
-I$(LIB_SRC)/finiteVolume/lnInclude \
-I$(LIB_SRC)/dynamicMesh/lnInclude \
-I$(LIB_SRC)/lagrangian/basic/lnInclude \
-I$(LIB_SRC)/meshTools/lnInclude \
-I$(LIB_SRC)/regionModels/regionModel/lnInclude
@ -12,5 +13,6 @@ EXE_LIBS = \
-lgenericPatchFields \
-ldecompositionMethods -L$(FOAM_LIBBIN)/dummy -lmetisDecomp -lscotchDecomp \
-llagrangian \
-ldynamicMesh \
-lmeshTools \
-lregionModels

90
applications/utilities/parallelProcessing/decomposePar/decomposePar.C
View File

@ -3,7 +3,7 @@
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2015 OpenFOAM Foundation
\\/ M anipulation |
\\/ M anipulation | Copyright (C) 2015 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -96,6 +96,7 @@ Usage
#include "fvFieldDecomposer.H"
#include "pointFieldDecomposer.H"
#include "lagrangianFieldDecomposer.H"
#include "decompositionModel.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
@ -142,6 +143,12 @@ int main(int argc, char *argv[])
);
argList::noParallel();
Foam::argList::addOption
(
"decomposeParDict",
"file",
"read decomposePar dictionary from specified location"
);
#include "addRegionOption.H"
argList::addBoolOption
(
@ -199,6 +206,17 @@ int main(int argc, char *argv[])
instantList times = timeSelector::selectIfPresent(runTime, args);
// Allow override of decomposeParDict location
fileName decompDictFile;
if (args.optionReadIfPresent("decomposeParDict", decompDictFile))
{
if (isDir(decompDictFile))
{
decompDictFile = decompDictFile/"decomposeParDict";
}
}
wordList regionNames;
wordList regionDirs;
if (allRegions)
@ -260,21 +278,27 @@ int main(int argc, char *argv[])
++nProcs;
}
// get requested numberOfSubdomains
// get requested numberOfSubdomains. Note: have no mesh yet so
// cannot use decompositionModel::New
const label nDomains = readLabel
(
IOdictionary
(
IOobject
decompositionModel::selectIO
(
"decomposeParDict",
runTime.time().system(),
regionDir, // use region if non-standard
runTime,
IOobject::MUST_READ_IF_MODIFIED,
IOobject::NO_WRITE,
false
IOobject
(
"decomposeParDict",
runTime.time().system(),
regionDir, // use region if non-standard
runTime,
IOobject::MUST_READ_IF_MODIFIED,
IOobject::NO_WRITE,
false
),
decompDictFile
)
).lookup("numberOfSubdomains")
);
@ -288,8 +312,7 @@ int main(int argc, char *argv[])
<< nProcs << " domains"
<< nl
<< "instead of " << nDomains
<< " domains as specified in decomposeParDict"
<< nl
<< " domains as specified in decomposeParDict" << nl
<< exit(FatalError);
}
}
@ -351,7 +374,8 @@ int main(int argc, char *argv[])
IOobject::NO_READ,
IOobject::NO_WRITE,
false
)
),
decompDictFile
);
// Decompose the mesh
@ -447,15 +471,15 @@ int main(int argc, char *argv[])
// Construct the vol fields
// ~~~~~~~~~~~~~~~~~~~~~~~~
PtrList<volScalarField> volScalarFields;
readFields(mesh, objects, volScalarFields);
readFields(mesh, objects, volScalarFields, false);
PtrList<volVectorField> volVectorFields;
readFields(mesh, objects, volVectorFields);
readFields(mesh, objects, volVectorFields, false);
PtrList<volSphericalTensorField> volSphericalTensorFields;
readFields(mesh, objects, volSphericalTensorFields);
readFields(mesh, objects, volSphericalTensorFields, false);
PtrList<volSymmTensorField> volSymmTensorFields;
readFields(mesh, objects, volSymmTensorFields);
readFields(mesh, objects, volSymmTensorFields, false);
PtrList<volTensorField> volTensorFields;
readFields(mesh, objects, volTensorFields);
readFields(mesh, objects, volTensorFields, false);
// Construct the dimensioned fields
@ -476,15 +500,15 @@ int main(int argc, char *argv[])
// Construct the surface fields
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
PtrList<surfaceScalarField> surfaceScalarFields;
readFields(mesh, objects, surfaceScalarFields);
readFields(mesh, objects, surfaceScalarFields, false);
PtrList<surfaceVectorField> surfaceVectorFields;
readFields(mesh, objects, surfaceVectorFields);
readFields(mesh, objects, surfaceVectorFields, false);
PtrList<surfaceSphericalTensorField> surfaceSphericalTensorFields;
readFields(mesh, objects, surfaceSphericalTensorFields);
readFields(mesh, objects, surfaceSphericalTensorFields, false);
PtrList<surfaceSymmTensorField> surfaceSymmTensorFields;
readFields(mesh, objects, surfaceSymmTensorFields);
readFields(mesh, objects, surfaceSymmTensorFields, false);
PtrList<surfaceTensorField> surfaceTensorFields;
readFields(mesh, objects, surfaceTensorFields);
readFields(mesh, objects, surfaceTensorFields, false);
// Construct the point fields
@ -492,15 +516,15 @@ int main(int argc, char *argv[])
const pointMesh& pMesh = pointMesh::New(mesh);
PtrList<pointScalarField> pointScalarFields;
readFields(pMesh, objects, pointScalarFields);
readFields(pMesh, objects, pointScalarFields, false);
PtrList<pointVectorField> pointVectorFields;
readFields(pMesh, objects, pointVectorFields);
readFields(pMesh, objects, pointVectorFields, false);
PtrList<pointSphericalTensorField> pointSphericalTensorFields;
readFields(pMesh, objects, pointSphericalTensorFields);
readFields(pMesh, objects, pointSphericalTensorFields, false);
PtrList<pointSymmTensorField> pointSymmTensorFields;
readFields(pMesh, objects, pointSymmTensorFields);
readFields(pMesh, objects, pointSymmTensorFields, false);
PtrList<pointTensorField> pointTensorFields;
readFields(pMesh, objects, pointTensorFields);
readFields(pMesh, objects, pointTensorFields, false);
// Construct the Lagrangian fields
@ -820,16 +844,6 @@ int main(int argc, char *argv[])
processorDb.setTime(runTime);
// remove files remnants that can cause horrible problems
// - mut and nut are used to mark the new turbulence models,
// their existence prevents old models from being upgraded
{
fileName timeDir(processorDb.path()/processorDb.timeName());
rm(timeDir/"mut");
rm(timeDir/"nut");
}
// read the mesh
if (!procMeshList.set(procI))
{

55
applications/utilities/parallelProcessing/decomposePar/decomposeParDict
View File

@ -17,11 +17,61 @@ FoamFile
numberOfSubdomains 2;
// Optional decomposition constraints
//constraints
//{
// preserveBaffles
// {
// //- Keep owner and neighbour of baffles on same processor (i.e.
// // keep it detectable as a baffle). Baffles are two boundary face
// // sharing the same points
// type preserveBaffles;
// }
// preserveFaceZones
// {
// //- Keep owner and neighbour on same processor for faces in zones
// type preserveFaceZones;
// zones (".*");
// }
// preservePatches
// {
// //- Keep owner and neighbour on same processor for faces in patches
// // (only makes sense for cyclic patches. Not suitable for e.g.
// // cyclicAMI since these are not coupled on the patch level. Use
// // singleProcessorFaceSets for those)
// type preservePatches;
// patches (".*");
// }
// singleProcessorFaceSets
// {
// //- Keep all of faceSet on a single processor. This puts all cells
// // connected with a point, edge or face on the same processor.
// // (just having face connected cells might not guarantee a balanced
// // decomposition)
// // The processor can be -1 (the decompositionMethod chooses the
// // processor for a good load balance) or explicitly provided (upsets
// // balance)
// type singleProcessorFaceSets;
// singleProcessorFaceSets ((f1 -1));
// }
// refinementHistory
// {
// //- Decompose cells such that all cell originating from single cell
// // end up on same processor
// type refinementHistory;
// }
//}
// Deprecated form of specifying decomposition constraints:
//- Keep owner and neighbour on same processor for faces in zones:
// preserveFaceZones (heater solid1 solid3);
//- Keep owner and neighbour on same processor for faces in patches:
// (makes sense only for cyclic patches)
// (makes sense only for cyclic patches. Not suitable for e.g. cyclicAMI
// since these are not coupled on the patch level. Use
// singleProcessorFaceSets for those)
//preservePatches (cyclic_half0 cyclic_half1);
//- Keep all of faceSet on a single processor. This puts all cells
@ -32,12 +82,13 @@ numberOfSubdomains 2;
// for a good load balance) or explicitly provided (upsets balance).
//singleProcessorFaceSets ((f0 -1));
//- Keep owner and neighbour of baffles on same processor (i.e. keep it
// detectable as a baffle). Baffles are two boundary face sharing the
// same points.
//preserveBaffles true;
//- Use the volScalarField named here as a weight for each cell in the
// decomposition. For example, use a particle population field to decompose
// for a balanced number of particles in a lagrangian simulation.

169
applications/utilities/parallelProcessing/decomposePar/domainDecomposition.C
View File

@ -3,7 +3,7 @@
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2015 OpenFOAM Foundation
\\/ M anipulation |
\\/ M anipulation | Copyright (C) 2015 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -31,14 +31,14 @@ License
#include "fvMesh.H"
#include "OSspecific.H"
#include "Map.H"
#include "globalMeshData.H"
#include "DynamicList.H"
#include "fvFieldDecomposer.H"
#include "IOobjectList.H"
#include "cellSet.H"
#include "faceSet.H"
#include "pointSet.H"
#include "uniformDimensionedFields.H"
#include "decompositionModel.H"
#include "hexRef8Data.H"
// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
@ -69,7 +69,12 @@ void Foam::domainDecomposition::mark
// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
Foam::domainDecomposition::domainDecomposition(const IOobject& io)
// from components
Foam::domainDecomposition::domainDecomposition
(
const IOobject& io,
const fileName& decompDictFile
)
:
fvMesh(io),
facesInstancePointsPtr_
@ -90,18 +95,18 @@ Foam::domainDecomposition::domainDecomposition(const IOobject& io)
)
: NULL
),
decompositionDict_
decompDictFile_(decompDictFile),
nProcs_
(
IOobject
readInt
(
"decomposeParDict",
time().system(),
*this,
IOobject::MUST_READ_IF_MODIFIED,
IOobject::NO_WRITE
decompositionModel::New
(
*this,
decompDictFile
).lookup("numberOfSubdomains")
)
),
nProcs_(readInt(decompositionDict_.lookup("numberOfSubdomains"))),
distributed_(false),
cellToProc_(nCells()),
procPointAddressing_(nProcs_),
@ -115,7 +120,11 @@ Foam::domainDecomposition::domainDecomposition(const IOobject& io)
procProcessorPatchSubPatchIDs_(nProcs_),
procProcessorPatchSubPatchStarts_(nProcs_)
{
decompositionDict_.readIfPresent("distributed", distributed_);
decompositionModel::New
(
*this,
decompDictFile
).readIfPresent("distributed", distributed_);
}
@ -195,57 +204,20 @@ bool Foam::domainDecomposition::writeDecomposition(const bool decomposeSets)
}
autoPtr<labelIOList> cellLevelPtr;
{
IOobject io
// Load refinement data (if any)
hexRef8Data baseMeshData
(
IOobject
(
"cellLevel",
"dummy",
facesInstance(),
polyMesh::meshSubDir,
*this,
IOobject::MUST_READ,
IOobject::NO_WRITE
);
if (io.headerOk())
{
Info<< "Reading hexRef8 data : " << io.name() << endl;
cellLevelPtr.reset(new labelIOList(io));
}
}
autoPtr<labelIOList> pointLevelPtr;
{
IOobject io
(
"pointLevel",
facesInstance(),
polyMesh::meshSubDir,
*this,
IOobject::MUST_READ,
IOobject::NO_WRITE
);
if (io.headerOk())
{
Info<< "Reading hexRef8 data : " << io.name() << endl;
pointLevelPtr.reset(new labelIOList(io));
}
}
autoPtr<uniformDimensionedScalarField> level0EdgePtr;
{
IOobject io
(
"level0Edge",
facesInstance(),
polyMesh::meshSubDir,
*this,
IOobject::MUST_READ,
IOobject::NO_WRITE
);
if (io.headerOk())
{
Info<< "Reading hexRef8 data : " << io.name() << endl;
level0EdgePtr.reset(new uniformDimensionedScalarField(io));
}
}
IOobject::READ_IF_PRESENT,
IOobject::NO_WRITE,
false
)
);
@ -816,8 +788,8 @@ bool Foam::domainDecomposition::writeDecomposition(const bool decomposeSets)
}
}
// Set the precision of the points data to 10
IOstream::defaultPrecision(10);
// Set the precision of the points data to be min 10
IOstream::defaultPrecision(max(10u, IOstream::defaultPrecision()));
procMesh.write();
@ -887,64 +859,23 @@ bool Foam::domainDecomposition::writeDecomposition(const bool decomposeSets)
}
// hexRef8 data
if (cellLevelPtr.valid())
{
labelIOList
// Optional hexRef8 data
hexRef8Data
(
IOobject
(
IOobject
(
cellLevelPtr().name(),
facesInstance(),
polyMesh::meshSubDir,
procMesh,
IOobject::NO_READ,
IOobject::AUTO_WRITE
),
UIndirectList<label>
(
cellLevelPtr(),
procCellAddressing_[procI]
)()
).write();
}
if (pointLevelPtr.valid())
{
labelIOList
(
IOobject
(
pointLevelPtr().name(),
facesInstance(),
polyMesh::meshSubDir,
procMesh,
IOobject::NO_READ,
IOobject::AUTO_WRITE
),
UIndirectList<label>
(
pointLevelPtr(),
procPointAddressing_[procI]
)()
).write();
}
if (level0EdgePtr.valid())
{
uniformDimensionedScalarField
(
IOobject
(
level0EdgePtr().name(),
facesInstance(),
polyMesh::meshSubDir,
procMesh,
IOobject::NO_READ,
IOobject::AUTO_WRITE
),
level0EdgePtr()
).write();
}
"dummy",
facesInstance(),
polyMesh::meshSubDir,
procMesh,
IOobject::NO_READ,
IOobject::NO_WRITE,
false
),
baseMeshData,
procCellAddressing_[procI],
procPointAddressing_[procI]
).write();
// Statistics

15
applications/utilities/parallelProcessing/decomposePar/domainDecomposition.H
View File

@ -3,7 +3,7 @@
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2014 OpenFOAM Foundation
\\/ M anipulation |
\\/ M anipulation | Copyright (C) 2015 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -61,8 +61,8 @@ class domainDecomposition
//- Optional: points at the facesInstance
autoPtr<pointIOField> facesInstancePointsPtr_;
//- Mesh decomposition control dictionary
IOdictionary decompositionDict_;
//- Optional non-standard file for decomposeParDict
const fileName decompDictFile_;
//- Number of processors in decomposition
label nProcs_;
@ -159,8 +159,13 @@ public:
// Constructors
//- Construct from IOobject
domainDecomposition(const IOobject& io);
//- Construct from IOobjects (for mesh and optional non-standard
// decomposeParDict location)
domainDecomposition
(
const IOobject& io,
const fileName& decompDictFile = ""
);
//- Destructor

15
applications/utilities/parallelProcessing/decomposePar/domainDecompositionDistribute.C
View File

@ -3,7 +3,7 @@
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
\\/ M anipulation | Copyright (C) 2015 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -30,6 +30,7 @@ License
#include "regionSplit.H"
#include "Tuple2.H"
#include "faceSet.H"
#include "decompositionModel.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
@ -39,15 +40,17 @@ void Foam::domainDecomposition::distributeCells()
cpuTime decompositionTime;
autoPtr<decompositionMethod> decomposePtr = decompositionMethod::New
const decompositionModel& method = decompositionModel::New
(
decompositionDict_
*this,
decompDictFile_
);
scalarField cellWeights;
if (decompositionDict_.found("weightField"))
if (method.found("weightField"))
{
word weightName = decompositionDict_.lookup("weightField");
word weightName = method.lookup("weightField");
volScalarField weights
(
@ -64,7 +67,7 @@ void Foam::domainDecomposition::distributeCells()
cellWeights = weights.internalField();
}
cellToProc_ = decomposePtr().decompose(*this, cellWeights);
cellToProc_ = method.decomposer().decompose(*this, cellWeights);
Info<< "\nFinished decomposition in "
<< decompositionTime.elapsedCpuTime()

65
applications/utilities/parallelProcessing/decomposePar/readFields.C
View File

@ -3,7 +3,7 @@
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011 OpenFOAM Foundation
\\/ M anipulation |
\\/ M anipulation | Copyright (C) 2015 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -23,10 +23,48 @@ License
\*---------------------------------------------------------------------------*/
#include "GeometricField.H"
#include "readFields.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
template<class Type, template<class> class PatchField, class GeoMesh>
void Foam::readFields
(
const typename GeoMesh::Mesh& mesh,
const IOobjectList& objects,
PtrList<GeometricField<Type, PatchField, GeoMesh> >& fields,
const bool readOldTime
)
{
typedef GeometricField<Type, PatchField, GeoMesh> GeoField;
// Search list of objects for fields of type GeomField
IOobjectList fieldObjects(objects.lookupClass(GeoField::typeName));
// Remove the cellDist field
IOobjectList::iterator celDistIter = fieldObjects.find("cellDist");
if (celDistIter != fieldObjects.end())
{
fieldObjects.erase(celDistIter);
}
// Get sorted set of names (different processors might read objects in
// different order)
const wordList masterNames(fieldObjects.sortedNames());
// Construct the fields
fields.setSize(masterNames.size());
forAll(masterNames, i)
{
const IOobject& io = *fieldObjects[masterNames[i]];
fields.set(i, new GeoField(io, mesh, readOldTime));
}
}
template<class Mesh, class GeoField>
void Foam::readFields
(
@ -38,24 +76,21 @@ void Foam::readFields
// Search list of objects for fields of type GeomField
IOobjectList fieldObjects(objects.lookupClass(GeoField::typeName));
// Remove the cellDist field
IOobjectList::iterator celDistIter = fieldObjects.find("cellDist");
if (celDistIter != fieldObjects.end())
{
fieldObjects.erase(celDistIter);
}
// Construct the fields
fields.setSize(fieldObjects.size());
label fieldI = 0;
forAllIter(IOobjectList, fieldObjects, iter)
// Get sorted set of names (different processors might read objects in
// different order)
const wordList masterNames(fieldObjects.sortedNames());
// Construct the fields
fields.setSize(masterNames.size());
forAll(masterNames, i)
{
fields.set
(
fieldI++,
new GeoField(*iter(), mesh)
);
const IOobject& io = *fieldObjects[masterNames[i]];
fields.set(i, new GeoField(io, mesh));
}
}

12
applications/utilities/parallelProcessing/decomposePar/readFields.H
View File

@ -3,7 +3,7 @@
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2015 OpenFOAM Foundation
\\/ M anipulation |
\\/ M anipulation | Copyright (C) 2015 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -41,6 +41,16 @@ SourceFiles
namespace Foam
{
// Read the fields and hold on the pointer list
template<class Type, template<class> class PatchField, class GeoMesh>
void readFields
(
const typename GeoMesh::Mesh& mesh,
const IOobjectList& objects,
PtrList<GeometricField<Type, PatchField, GeoMesh> >& fields,
const bool readOldTime
);
// Read the fields and hold on the pointer list
template<class Mesh, class GeoField>
void readFields