zhyang-dev/OpenFOAM-12
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

parallelProcessing: Various improvements

Browse Source 
boundaryProcAddressing has been removed. This has not been needed for a
long time. decomposePar has been optimised for mininum IO, rather than
minimum memory usage. decomposePar has also been corrected so that it
can decompose sequences of time-varying meshes.
This commit is contained in:
Will Bainbridge
2022-03-10 11:24:21 +00:00
parent dc052dd20c
commit 3995456979
26 changed files with 1582 additions and 1808 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
applications/utilities/parallelProcessing/decomposePar/Make/files
View File

@ -1,7 +1,5 @@
decomposePar.C
domainDecomposition.C
domainDecompositionMesh.C
domainDecompositionDistribute.C
dimFieldDecomposer.C
fvFieldDecomposer.C
pointFieldDecomposer.C

616
applications/utilities/parallelProcessing/decomposePar/decomposePar.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2011-2021 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2022 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -111,49 +111,15 @@ using namespace Foam;
namespace Foam
{
const labelIOList& procAddressing
(
const PtrList<fvMesh>& procMeshList,
const label proci,
const word& name,
PtrList<labelIOList>& procAddressingList
)
{
const fvMesh& procMesh = procMeshList[proci];
if (!procAddressingList.set(proci))
{
procAddressingList.set
(
proci,
new labelIOList
(
IOobject
(
name,
procMesh.facesInstance(),
procMesh.meshSubDir,
procMesh,
IOobject::MUST_READ,
IOobject::NO_WRITE,
false
)
)
);
}
return procAddressingList[proci];
}
void decomposeUniform
(
const bool copyUniform,
const domainDecomposition& mesh,
const domainDecomposition& decomposition,
const Time& processorDb,
const word& regionDir = word::null
)
{
const Time& runTime = mesh.time();
const Time& runTime = decomposition.mesh().time();
// Any uniform data to copy/link?
const fileName uniformDir(regionDir/"uniform");
@ -167,7 +133,7 @@ void decomposeUniform
const fileName timePath =
fileHandler().filePath(processorDb.timePath());
if (copyUniform || mesh.distributed())
if (copyUniform || decomposition.distributed())
{
if (!fileHandler().exists(timePath/uniformDir))
{
@ -204,6 +170,57 @@ void decomposeUniform
}
}
void writeDecomposition(const domainDecomposition& decomposition)
{
const labelList& procIds = decomposition.cellToProc();
// Write the decomposition as labelList for use with 'manual'
// decomposition method.
labelIOList cellDecomposition
(
IOobject
(
"cellDecomposition",
decomposition.mesh().facesInstance(),
decomposition.mesh(),
IOobject::NO_READ,
IOobject::NO_WRITE,
false
),
procIds
);
cellDecomposition.write();
Info<< nl << "Wrote decomposition to "
<< cellDecomposition.relativeObjectPath()
<< " for use in manual decomposition." << endl;
// Write as volScalarField for postprocessing.
volScalarField::Internal cellDist
(
IOobject
(
"cellDist",
decomposition.mesh().time().timeName(),
decomposition.mesh(),
IOobject::NO_READ,
IOobject::AUTO_WRITE
),
decomposition.mesh(),
dimless,
scalarField(scalarList(procIds))
);
cellDist.write();
Info<< nl << "Wrote decomposition as volScalarField to "
<< cellDist.name() << " for use in postprocessing."
<< endl;
}
}
@ -291,17 +308,18 @@ int main(int argc, char *argv[])
}
}
// Set time from database
#include "createTime.H"
// Allow override of time
instantList times = timeSelector::selectIfPresent(runTime, args);
// Get region names
const wordList regionNames(selectRegionNames(args, runTime));
// Handle existing decomposition directories
{
// Determine the existing processor count directly
// Determine the processor count from the directories
label nProcs = fileHandler().nProcs(runTime.path());
if (forceOverwrite)
@ -367,7 +385,7 @@ int main(int argc, char *argv[])
}
}
// Decompose all regions
forAll(regionNames, regioni)
{
const word& regionName = regionNames[regioni];
@ -386,33 +404,25 @@ int main(int argc, char *argv[])
// Give file handler a chance to determine the output directory
const_cast<fileOperation&>(fileHandler()).setNProcs(nDomains);
if (decomposeFieldsOnly)
// Sanity check number of processors in a previously decomposed case
if (decomposeFieldsOnly && nProcs != nDomains)
{
// Sanity check on previously decomposed case
if (nProcs != nDomains)
{
FatalErrorInFunction
<< "Specified -fields, but the case was decomposed with "
<< nProcs << " domains"
<< nl
<< "instead of " << nDomains
<< " domains as specified in decomposeParDict"
<< nl
<< exit(FatalError);
}
FatalErrorInFunction
<< "Specified -fields, but the case was decomposed with "
<< nProcs << " domains" << nl << "instead of " << nDomains
<< " domains as specified in decomposeParDict" << nl
<< exit(FatalError);
}
else if (nProcs)
// Reuse the decomposition if permitted
if (ifRequiredDecomposition && nProcs == nDomains)
{
if (ifRequiredDecomposition && nProcs == nDomains)
{
// Reuse the decomposition
decomposeFieldsOnly = true;
Info<< "Using existing processor directories" << nl;
}
decomposeFieldsOnly = true;
Info<< "Using existing processor directories" << nl;
}
Info<< "Create mesh" << endl;
domainDecomposition mesh
fvMesh mesh
(
IOobject
(
@ -422,148 +432,151 @@ int main(int argc, char *argv[])
IOobject::NO_READ,
IOobject::NO_WRITE,
false
)
),
false
);
// Decompose the mesh
if (!decomposeFieldsOnly)
// Create decomposition
domainDecomposition decomposition(mesh);
// Read or generate a decomposition as necessary
if (decomposeFieldsOnly)
{
mesh.decomposeMesh();
mesh.writeDecomposition(decomposeSets);
decomposition.read();
}
else
{
decomposition.decompose();
decomposition.write(decomposeSets);
if (writeCellDist)
{
const labelList& procIds = mesh.cellToProc();
// Write the decomposition as labelList for use with 'manual'
// decomposition method.
labelIOList cellDecomposition
(
IOobject
(
"cellDecomposition",
mesh.facesInstance(),
mesh,
IOobject::NO_READ,
IOobject::NO_WRITE,
false
),
procIds
);
cellDecomposition.write();
Info<< nl << "Wrote decomposition to "
<< cellDecomposition.relativeObjectPath()
<< " for use in manual decomposition." << endl;
// Write as volScalarField for postprocessing.
volScalarField cellDist
(
IOobject
(
"cellDist",
runTime.timeName(),
mesh,
IOobject::NO_READ,
IOobject::AUTO_WRITE
),
mesh,
dimensionedScalar(dimless, 0)
);
forAll(procIds, celli)
{
cellDist[celli] = procIds[celli];
}
cellDist.write();
Info<< nl << "Wrote decomposition as volScalarField to "
<< cellDist.name() << " for use in postprocessing."
<< endl;
writeDecomposition(decomposition);
}
fileHandler().flush();
}
// Field maps. These are preserved if decomposing multiple times.
PtrList<fvFieldDecomposer> fieldDecomposerList
(
decomposition.nProcs()
);
PtrList<dimFieldDecomposer> dimFieldDecomposerList
(
decomposition.nProcs()
);
PtrList<pointFieldDecomposer> pointFieldDecomposerList
(
decomposition.nProcs()
);
if (copyZero)
// Loop over all times
forAll(times, timeI)
{
// Copy the 0 directory into each of the processor directories
fileName prevTimePath;
for (label proci = 0; proci < mesh.nProcs(); proci++)
// Set the time
runTime.setTime(times[timeI], timeI);
decomposition.setTime(times[timeI], timeI);
Info<< "Time = " << runTime.userTimeName() << endl;
// Update the mesh
const fvMesh::readUpdateState state = mesh.readUpdate();
// Update the decomposition
if (decomposeFieldsOnly)
{
Time processorDb
(
Time::controlDictName,
args.rootPath(),
args.caseName()/fileName(word("processor") + name(proci))
);
processorDb.setTime(runTime);
if (fileHandler().isDir(runTime.timePath()))
decomposition.readUpdate();
}
else if (state == fvMesh::POINTS_MOVED)
{
decomposition.writePoints();
}
else if
(
state == fvMesh::TOPO_CHANGE
|| state == fvMesh::TOPO_PATCH_CHANGE
)
{
decomposition.decompose();
decomposition.write(decomposeSets);
if (writeCellDist)
{
// Get corresponding directory name (to handle processors/)
const fileName timePath
writeDecomposition(decomposition);
}
}
// Clear the field maps if there has been topology change
if
(
state == fvMesh::TOPO_CHANGE
|| state == fvMesh::TOPO_PATCH_CHANGE
)
{
for (label proci = 0; proci < decomposition.nProcs(); proci++)
{
fieldDecomposerList.set(proci, nullptr);
dimFieldDecomposerList.set(proci, nullptr);
pointFieldDecomposerList.set(proci, nullptr);
}
}
// Decompose the fields at this time
if (decomposeGeomOnly)
{
// Do nothing
}
else if (copyZero)
{
// Copy the field files from the <time> directory to the
// processor*/<time> directories without altering them
fileName prevTimePath;
for (label proci = 0; proci < decomposition.nProcs(); proci++)
{
Time processorDb
(
fileHandler().objectPath
(
IOobject
(
"",
processorDb.timeName(),
processorDb
),
word::null
)
Time::controlDictName,
args.rootPath(),
args.caseName()
/fileName(word("processor") + name(proci))
);
processorDb.setTime(runTime);
if (timePath != prevTimePath)
if (fileHandler().isDir(runTime.timePath()))
{
Info<< "Processor " << proci
<< ": copying " << runTime.timePath() << nl
<< " to " << timePath << endl;
fileHandler().cp(runTime.timePath(), timePath);
const fileName timePath
(
fileHandler().objectPath
(
IOobject
(
"",
processorDb.timeName(),
processorDb
),
word::null
)
);
prevTimePath = timePath;
if (timePath != prevTimePath)
{
Info<< "Processor " << proci
<< ": copying " << runTime.timePath() << nl
<< " to " << timePath << endl;
fileHandler().cp(runTime.timePath(), timePath);
prevTimePath = timePath;
}
}
}
}
}
else if (!decomposeGeomOnly)
{
// Decompose the field files
// Cached processor meshes and maps. These are only preserved if
// running with multiple times.
PtrList<Time> processorDbList(mesh.nProcs());
PtrList<fvMesh> procMeshList(mesh.nProcs());
PtrList<labelIOList> faceProcAddressingList(mesh.nProcs());
PtrList<labelIOList> cellProcAddressingList(mesh.nProcs());
PtrList<labelIOList> boundaryProcAddressingList(mesh.nProcs());
PtrList<fvFieldDecomposer> fieldDecomposerList(mesh.nProcs());
PtrList<dimFieldDecomposer> dimFieldDecomposerList(mesh.nProcs());
PtrList<labelIOList> pointProcAddressingList(mesh.nProcs());
PtrList<pointFieldDecomposer> pointFieldDecomposerList
(
mesh.nProcs()
);
// Loop over all times
forAll(times, timeI)
else
{
runTime.setTime(times[timeI], timeI);
Info<< "Time = " << runTime.userTimeName() << endl;
// Decompose the fields
// Search for list of objects for this time
IOobjectList objects(mesh, runTime.timeName());
// Construct the vol fields
// ~~~~~~~~~~~~~~~~~~~~~~~~
PtrList<volScalarField> volScalarFields;
readFields(mesh, objects, volScalarFields);
PtrList<volVectorField> volVectorFields;
@ -575,9 +588,7 @@ int main(int argc, char *argv[])
PtrList<volTensorField> volTensorFields;
readFields(mesh, objects, volTensorFields);
// Construct the dimensioned fields
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
PtrList<DimensionedField<scalar, volMesh>> dimScalarFields;
readFields(mesh, objects, dimScalarFields);
PtrList<DimensionedField<vector, volMesh>> dimVectorFields;
@ -591,9 +602,7 @@ int main(int argc, char *argv[])
PtrList<DimensionedField<tensor, volMesh>> dimTensorFields;
readFields(mesh, objects, dimTensorFields);
// Construct the surface fields
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
PtrList<surfaceScalarField> surfaceScalarFields;
readFields(mesh, objects, surfaceScalarFields);
PtrList<surfaceVectorField> surfaceVectorFields;
@ -606,11 +615,8 @@ int main(int argc, char *argv[])
PtrList<surfaceTensorField> surfaceTensorFields;
readFields(mesh, objects, surfaceTensorFields);
// Construct the point fields
// ~~~~~~~~~~~~~~~~~~~~~~~~~~
const pointMesh& pMesh = pointMesh::New(mesh);
PtrList<pointScalarField> pointScalarFields;
readFields(pMesh, objects, pointScalarFields);
PtrList<pointVectorField> pointVectorFields;
@ -622,10 +628,7 @@ int main(int argc, char *argv[])
PtrList<pointTensorField> pointTensorFields;
readFields(pMesh, objects, pointTensorFields);
// Construct the Lagrangian fields
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
fileNameList cloudDirs
(
fileHandler().readDir
@ -634,70 +637,34 @@ int main(int argc, char *argv[])
fileType::directory
)
);
// Particles
PtrList<Cloud<indexedParticle>> lagrangianPositions
(
cloudDirs.size()
);
// Particles per cell
PtrList<List<SLList<indexedParticle*>*>> cellParticles
(
cloudDirs.size()
);
PtrList<PtrList<labelIOField>> lagrangianLabelFields
(
cloudDirs.size()
);
PtrList<Cloud<indexedParticle>>
lagrangianPositions(cloudDirs.size());
PtrList<List<SLList<indexedParticle*>*>>
cellParticles(cloudDirs.size());
PtrList<PtrList<labelIOField>>
lagrangianLabelFields(cloudDirs.size());
PtrList<PtrList<labelFieldCompactIOField>>
lagrangianLabelFieldFields
(
cloudDirs.size()
);
PtrList<PtrList<scalarIOField>> lagrangianScalarFields
(
cloudDirs.size()
);
lagrangianLabelFieldFields(cloudDirs.size());
PtrList<PtrList<scalarIOField>>
lagrangianScalarFields(cloudDirs.size());
PtrList<PtrList<scalarFieldCompactIOField>>
lagrangianScalarFieldFields
(
cloudDirs.size()
);
PtrList<PtrList<vectorIOField>> lagrangianVectorFields
(
cloudDirs.size()
);
lagrangianScalarFieldFields(cloudDirs.size());
PtrList<PtrList<vectorIOField>>
lagrangianVectorFields(cloudDirs.size());
PtrList<PtrList<vectorFieldCompactIOField>>
lagrangianVectorFieldFields
(
cloudDirs.size()
);
lagrangianVectorFieldFields(cloudDirs.size());
PtrList<PtrList<sphericalTensorIOField>>
lagrangianSphericalTensorFields
(
cloudDirs.size()
);
lagrangianSphericalTensorFields(cloudDirs.size());
PtrList<PtrList<sphericalTensorFieldCompactIOField>>
lagrangianSphericalTensorFieldFields(cloudDirs.size());
PtrList<PtrList<symmTensorIOField>> lagrangianSymmTensorFields
(
cloudDirs.size()
);
PtrList<PtrList<symmTensorIOField>>
lagrangianSymmTensorFields(cloudDirs.size());
PtrList<PtrList<symmTensorFieldCompactIOField>>
lagrangianSymmTensorFieldFields
(
cloudDirs.size()
);
PtrList<PtrList<tensorIOField>> lagrangianTensorFields
(
cloudDirs.size()
);
lagrangianSymmTensorFieldFields(cloudDirs.size());
PtrList<PtrList<tensorIOField>>
lagrangianTensorFields(cloudDirs.size());
PtrList<PtrList<tensorFieldCompactIOField>>
lagrangianTensorFieldFields
(
cloudDirs.size()
);
lagrangianTensorFieldFields(cloudDirs.size());
label cloudI = 0;
@ -721,11 +688,8 @@ int main(int argc, char *argv[])
if (positionsPtr)
{
// Read lagrangian particles
// ~~~~~~~~~~~~~~~~~~~~~~~~~
Info<< "Identified lagrangian data set: "
<< cloudDirs[i] << endl;
lagrangianPositions.set
(
cloudI,
@ -737,10 +701,7 @@ int main(int argc, char *argv[])
)
);
// Sort particles per cell
// ~~~~~~~~~~~~~~~~~~~~~~~
cellParticles.set
(
cloudI,
@ -751,8 +712,8 @@ int main(int argc, char *argv[])
)
);
label i = 0;
// Populate the cloud
label index = 0;
forAllIter
(
Cloud<indexedParticle>,
@ -760,7 +721,7 @@ int main(int argc, char *argv[])
iter
)
{
iter().index() = i++;
iter().index() = index ++;
label celli = iter().cell();
@ -791,8 +752,6 @@ int main(int argc, char *argv[])
}
// Read fields
// ~~~~~~~~~~~
IOobjectList lagrangianObjects
(
mesh,
@ -802,84 +761,72 @@ int main(int argc, char *argv[])
IOobject::NO_WRITE,
false
);
lagrangianFieldDecomposer::readFields
(
cloudI,
lagrangianObjects,
lagrangianLabelFields
);
lagrangianFieldDecomposer::readFieldFields
(
cloudI,
lagrangianObjects,
lagrangianLabelFieldFields
);
lagrangianFieldDecomposer::readFields
(
cloudI,
lagrangianObjects,
lagrangianScalarFields
);
lagrangianFieldDecomposer::readFieldFields
(
cloudI,
lagrangianObjects,
lagrangianScalarFieldFields
);
lagrangianFieldDecomposer::readFields
(
cloudI,
lagrangianObjects,
lagrangianVectorFields
);
lagrangianFieldDecomposer::readFieldFields
(
cloudI,
lagrangianObjects,
lagrangianVectorFieldFields
);
lagrangianFieldDecomposer::readFields
(
cloudI,
lagrangianObjects,
lagrangianSphericalTensorFields
);
lagrangianFieldDecomposer::readFieldFields
(
cloudI,
lagrangianObjects,
lagrangianSphericalTensorFieldFields
);
lagrangianFieldDecomposer::readFields
(
cloudI,
lagrangianObjects,
lagrangianSymmTensorFields
);
lagrangianFieldDecomposer::readFieldFields
(
cloudI,
lagrangianObjects,
lagrangianSymmTensorFieldFields
);
lagrangianFieldDecomposer::readFields
(
cloudI,
lagrangianObjects,
lagrangianTensorFields
);
lagrangianFieldDecomposer::readFieldFields
(
cloudI,
@ -909,76 +856,10 @@ int main(int argc, char *argv[])
Info<< endl;
// split the fields over processors
for (label proci = 0; proci < mesh.nProcs(); proci++)
for (label proci = 0; proci < decomposition.nProcs(); proci++)
{
Info<< "Processor " << proci << ": field transfer" << endl;
// open the database
if (!processorDbList.set(proci))
{
processorDbList.set
(
proci,
new Time
(
Time::controlDictName,
args.rootPath(),
args.caseName()
/fileName(word("processor") + name(proci))
)
);
}
Time& processorDb = processorDbList[proci];
processorDb.setTime(runTime);
// read the mesh
if (!procMeshList.set(proci))
{
procMeshList.set
(
proci,
new fvMesh
(
IOobject
(
regionName,
processorDb.timeName(),
processorDb
),
false
)
);
}
const fvMesh& procMesh = procMeshList[proci];
const labelIOList& faceProcAddressing = procAddressing
(
procMeshList,
proci,
"faceProcAddressing",
faceProcAddressingList
);
const labelIOList& cellProcAddressing = procAddressing
(
procMeshList,
proci,
"cellProcAddressing",
cellProcAddressingList
);
const labelIOList& boundaryProcAddressing = procAddressing
(
procMeshList,
proci,
"boundaryProcAddressing",
boundaryProcAddressingList
);
// FV fields
{
if (!fieldDecomposerList.set(proci))
@ -989,10 +870,9 @@ int main(int argc, char *argv[])
new fvFieldDecomposer
(
mesh,
procMesh,
faceProcAddressing,
cellProcAddressing,
boundaryProcAddressing
decomposition.procMeshes()[proci],
decomposition.procFaceAddressing()[proci],
decomposition.procCellAddressing()[proci]
)
);
}
@ -1037,9 +917,9 @@ int main(int argc, char *argv[])
new dimFieldDecomposer
(
mesh,
procMesh,
faceProcAddressing,
cellProcAddressing
decomposition.procMeshes()[proci],
decomposition.procFaceAddressing()[proci],
decomposition.procCellAddressing()[proci]
)
);
}
@ -1058,7 +938,6 @@ int main(int argc, char *argv[])
}
}
// Point fields
if
(
@ -1069,15 +948,8 @@ int main(int argc, char *argv[])
|| pointTensorFields.size()
)
{
const labelIOList& pointProcAddressing = procAddressing
(
procMeshList,
proci,
"pointProcAddressing",
pointProcAddressingList
);
const pointMesh& procPMesh = pointMesh::New(procMesh);
const pointMesh& procPMesh =
pointMesh::New(decomposition.procMeshes()[proci]);
if (!pointFieldDecomposerList.set(proci))
{
@ -1088,8 +960,7 @@ int main(int argc, char *argv[])
(
pMesh,
procPMesh,
pointProcAddressing,
boundaryProcAddressing
decomposition.procPointAddressing()[proci]
)
);
}
@ -1105,15 +976,12 @@ int main(int argc, char *argv[])
pointDecomposer.decomposeFields(pointSymmTensorFields);
pointDecomposer.decomposeFields(pointTensorFields);
if (times.size() == 1)
{
pointProcAddressingList.set(proci, nullptr);
pointFieldDecomposerList.set(proci, nullptr);
}
}
// If there is lagrangian data write it out
forAll(lagrangianPositions, cloudI)
{
@ -1122,9 +990,9 @@ int main(int argc, char *argv[])
lagrangianFieldDecomposer fieldDecomposer
(
mesh,
procMesh,
faceProcAddressing,
cellProcAddressing,
decomposition.procMeshes()[proci],
decomposition.procFaceAddressing()[proci],
decomposition.procCellAddressing()[proci],
cloudDirs[cloudI],
lagrangianPositions[cloudI],
cellParticles[cloudI]
@ -1196,27 +1064,27 @@ int main(int argc, char *argv[])
}
}
// Decompose the "uniform" directory in the time region
// Decompose the "uniform" directory in the region time
// directory
decomposeUniform(copyUniform, mesh, processorDb, regionDir);
decomposeUniform
(
copyUniform,
mesh,
decomposition.procMeshes()[proci].time(),
regionDir
);
// For the first region of a multi-region case additionally
// decompose the "uniform" directory in the time directory
// decompose the "uniform" directory in the no-region time
// directory
if (regionNames.size() > 1 && regioni == 0)
{
decomposeUniform(copyUniform, mesh, processorDb);
}
// We have cached all the constant mesh data for the current
// processor. This is only important if running with
// multiple times, otherwise it is just extra storage.
if (times.size() == 1)
{
boundaryProcAddressingList.set(proci, nullptr);
cellProcAddressingList.set(proci, nullptr);
faceProcAddressingList.set(proci, nullptr);
procMeshList.set(proci, nullptr);
processorDbList.set(proci, nullptr);
decomposeUniform
(
copyUniform,
mesh,
decomposition.procMeshes()[proci].time()
);
}
}
}

1426
applications/utilities/parallelProcessing/decomposePar/domainDecomposition.C
View File

File diff suppressed because it is too large Load Diff

177
applications/utilities/parallelProcessing/decomposePar/domainDecomposition.H
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2011-2021 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2022 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -37,13 +37,9 @@ SourceFiles
#define domainDecomposition_H
#include "fvMesh.H"
#include "labelList.H"
#include "SLList.H"
#include "PtrList.H"
#include "point.H"
#include "Time.H"
#include "volFields.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
namespace Foam
{
@ -53,68 +49,54 @@ namespace Foam
\*---------------------------------------------------------------------------*/
class domainDecomposition
:
public fvMesh
{
// Private Data
//- Reference to the complete mesh
const fvMesh& mesh_;
//- Number of processors in decomposition
const label nProcs_;
//- Optional: points at the facesInstance
autoPtr<pointIOField> facesInstancePointsPtr_;
//- Number of processors in decomposition
label nProcs_;
//- Is the decomposition data to be distributed for each processor
bool distributed_;
//- Processor label for each cell
labelList cellToProc_;
//- Labels of points for each processor
labelListList procPointAddressing_;
// Processor mesh to complete mesh addressing
//- Labels of faces for each processor
// Note: Face turning index is stored as the sign on addressing
// Only the processor boundary faces are affected: if the sign of the
// index is negative, the processor face is the reverse of the
// original face. In order to do this properly, all face
// indices will be incremented by 1 and the decremented as
// necessary to avoid the problem of face number zero having no
// sign.
List<DynamicList<label>> procFaceAddressing_;
//- Labels of points for each processor
labelListList procPointAddressing_;
//- Labels of cells for each processor
labelListList procCellAddressing_;
//- Labels of faces for each processor
// Note: Face turning index is stored as the sign on addressing
// Only the processor boundary faces are affected: if the sign of
// the index is negative, the processor face is the reverse of the
// original face. In order to do this properly, all face
// indices will be incremented by 1 and the decremented as
// necessary to avoid the problem of face number zero having no
// sign.
List<DynamicList<label>> procFaceAddressing_;
//- Sizes for processor mesh patches
// Excludes inter-processor boundaries
labelListList procPatchSize_;
//- Start indices for processor patches
// Excludes inter-processor boundaries
labelListList procPatchStartIndex_;
//- Labels of cells for each processor
labelListList procCellAddressing_;
// Per inter-processor patch information
//- Processor times
PtrList<Time> procRunTimes_;
//- Neighbour processor ID for inter-processor boundaries
labelListList procNeighbourProcessors_;
//- Processor meshes
PtrList<fvMesh> procMeshes_;
//- Sizes for inter-processor patches
labelListList procProcessorPatchSize_;
//- Start indices (in procFaceAddressing_) for inter-processor patches
labelListList procProcessorPatchStartIndex_;
//- Sub patch IDs for inter-processor patches
List<labelListList> procProcessorPatchSubPatchIDs_;
//- Sub patch sizes for inter-processor patches
List<labelListList> procProcessorPatchSubPatchStarts_;
// Private Member Functions
void distributeCells();
//- Call the decomposition method and return the processor index that
// each cell is being distributed to
labelList distributeCells();
//- Mark all elements with value or -2 if occur twice
static void mark
@ -124,24 +106,21 @@ class domainDecomposition
labelList& elementToZone
);
//- Append single element to list
static void append(labelList&, const label);
//- Add face to inter-processor patch
void addInterProcFace
(
const label facei,
const label ownerProc,
const label nbrProc,
List<Map<label>>&,
List<DynamicList<DynamicList<label>>>&
) const;
//- Generate sub patch info for processor cyclics
template<class BinaryOp>
void processInterCyclics
inline void processInterCyclics
(
const labelList& cellToProc,
const polyBoundaryMesh& patches,
List<DynamicList<DynamicList<label>>>& interPatchFaces,
List<Map<label>>& procNbrToInterPatch,
@ -151,44 +130,103 @@ class domainDecomposition
BinaryOp bop
) const;
//- Validate that the decomposition has been generated or read
void validate() const;
public:
//- Runtime type information
TypeName("domainDecomposition");
// Constructors
//- Construct from IOobject
domainDecomposition(const IOobject& io);
//- Construct from reference to fvMesh
domainDecomposition(const fvMesh& mesh);
//- Destructor
~domainDecomposition();
virtual ~domainDecomposition();
// Member Functions
//- Number of processor in decomposition
//- Access the global mesh
const fvMesh& mesh() const
{
return mesh_;
}
//- Return the number of processors in the decomposition
label nProcs() const
{
return nProcs_;
}
//- Is the decomposition data to be distributed for each processor
//- Is the decomposition data to be distributed for each processor?
bool distributed() const
{
return distributed_;
}
//- Decompose mesh
void decomposeMesh();
//- Write decomposition
bool writeDecomposition(const bool decomposeSets);
//- Cell-processor decomposition labels
const labelList& cellToProc() const
//- Access the labels of points for each processor
const labelListList& procPointAddressing() const
{
return cellToProc_;
validate();
return procPointAddressing_;
}
//- Access the labels of faces for each processor (see notes above)
const List<DynamicList<label>>& procFaceAddressing() const
{
validate();
return procFaceAddressing_;
}
//- Access the labels of cells for each processor
const labelListList& procCellAddressing() const
{
validate();
return procCellAddressing_;
}
//- Return the processor meshes
const PtrList<fvMesh>& procMeshes() const
{
validate();
return procMeshes_;
}
//- Generate the decomposition
void decompose();
//- Reset the time
void setTime(const instant&, const label newIndex);
//- Read additional (faceInstance) points, if necessary
void readPoints();
//- Read in the decomposition addressing
void readAddressing();
//- Read in the meshes and the decomposition
void read();
//- Read update the meshes and the decomposition
fvMesh::readUpdateState readUpdate();
//- Return the distribution as an FV field for writing
labelList cellToProc() const;
//- Read additional (faceInstance) points, if necessary
void writePoints() const;
//- Write the decomposition addressing
void writeAddressing() const;
//- Write the decomposed meshes and associated data
void write(const bool decomposeSets) const;
};
@ -196,13 +234,6 @@ public:
} // End namespace Foam
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
#ifdef NoRepository
#include "domainDecompositionTemplates.C"
#endif
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
#endif

71
applications/utilities/parallelProcessing/decomposePar/domainDecompositionDistribute.C
View File

@ -1,71 +0,0 @@
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2011-2021 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
\*---------------------------------------------------------------------------*/
#include "domainDecomposition.H"
#include "decompositionMethod.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
void Foam::domainDecomposition::distributeCells()
{
Info<< "\nCalculating distribution of cells" << endl;
cpuTime decompositionTime;
const dictionary decomposeParDict
(
decompositionMethod::decomposeParDict(time())
);
scalarField cellWeights;
if (decomposeParDict.found("weightField"))
{
const word weightName = decomposeParDict.lookup("weightField");
volScalarField weights
(
IOobject
(
weightName,
time().timeName(),
*this,
IOobject::MUST_READ,
IOobject::NO_WRITE
),
*this
);
cellWeights = weights.primitiveField();
}
cellToProc_ =
decompositionMethod::NewDecomposer(decomposeParDict)
->decompose(*this, cellWeights);
Info<< "\nFinished decomposition in "
<< decompositionTime.elapsedCpuTime()
<< " s" << endl;
}
// ************************************************************************* //

482
applications/utilities/parallelProcessing/decomposePar/domainDecompositionMesh.C
View File

@ -1,482 +0,0 @@
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2011-2021 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
InClass
domainDecomposition
Description
Private member of domainDecomposition.
Decomposes the mesh into bits
\*---------------------------------------------------------------------------*/
#include "domainDecomposition.H"
// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
void Foam::domainDecomposition::append(labelList& lst, const label elem)
{
label sz = lst.size();
lst.setSize(sz+1);
lst[sz] = elem;
}
void Foam::domainDecomposition::addInterProcFace
(
const label facei,
const label ownerProc,
const label nbrProc,
List<Map<label>>& nbrToInterPatch,
List<DynamicList<DynamicList<label>>>& interPatchFaces
) const
{
Map<label>::iterator patchiter = nbrToInterPatch[ownerProc].find(nbrProc);
// Introduce turning index only for internal faces (are duplicated).
label ownerIndex = facei+1;
label nbrIndex = -(facei+1);
if (patchiter != nbrToInterPatch[ownerProc].end())
{
// Existing interproc patch. Add to both sides.
label toNbrProcPatchi = patchiter();
interPatchFaces[ownerProc][toNbrProcPatchi].append(ownerIndex);
if (isInternalFace(facei))
{
label toOwnerProcPatchi = nbrToInterPatch[nbrProc][ownerProc];
interPatchFaces[nbrProc][toOwnerProcPatchi].append(nbrIndex);
}
}
else
{
// Create new interproc patches.
label toNbrProcPatchi = nbrToInterPatch[ownerProc].size();
nbrToInterPatch[ownerProc].insert(nbrProc, toNbrProcPatchi);
DynamicList<label> oneFace;
oneFace.append(ownerIndex);
interPatchFaces[ownerProc].append(oneFace);
if (isInternalFace(facei))
{
label toOwnerProcPatchi = nbrToInterPatch[nbrProc].size();
nbrToInterPatch[nbrProc].insert(ownerProc, toOwnerProcPatchi);
oneFace.clear();
oneFace.append(nbrIndex);
interPatchFaces[nbrProc].append(oneFace);
}
}
}
void Foam::domainDecomposition::decomposeMesh()
{
// Decide which cell goes to which processor
distributeCells();
// Distribute the cells according to the given processor label
// calculate the addressing information for the original mesh
Info<< "\nCalculating original mesh data" << endl;
// set references to the original mesh
const polyBoundaryMesh& patches = boundaryMesh();
const faceList& fcs = faces();
const labelList& owner = faceOwner();
const labelList& neighbour = faceNeighbour();
// loop through the list of processor labels for the cell and add the
// cell shape to the list of cells for the appropriate processor
Info<< "\nDistributing cells to processors" << endl;
// Cells per processor
procCellAddressing_ = invertOneToMany(nProcs_, cellToProc_);
Info<< "\nDistributing faces to processors" << endl;
// Loop through all internal faces and decide which processor they belong to
// First visit all internal faces. If cells at both sides belong to the
// same processor, the face is an internal face. If they are different,
// it belongs to both processors.
procFaceAddressing_.setSize(nProcs_);
// Internal faces
forAll(neighbour, facei)
{
if (cellToProc_[owner[facei]] == cellToProc_[neighbour[facei]])
{
// Face internal to processor. Notice no turning index.
procFaceAddressing_[cellToProc_[owner[facei]]].append(facei+1);
}
}
// for all processors, set the size of start index and patch size
// lists to the number of patches in the mesh
forAll(procPatchSize_, proci)
{
procPatchSize_[proci].setSize(patches.size());
procPatchStartIndex_[proci].setSize(patches.size());
}
forAll(patches, patchi)
{
// Reset size and start index for all processors
forAll(procPatchSize_, proci)
{
procPatchSize_[proci][patchi] = 0;
procPatchStartIndex_[proci][patchi] =
procFaceAddressing_[proci].size();
}
const label patchStart = patches[patchi].start();
if (!isA<cyclicPolyPatch>(patches[patchi]))
{
// Normal patch. Add faces to processor where the cell
// next to the face lives
const labelUList& patchFaceCells =
patches[patchi].faceCells();
forAll(patchFaceCells, facei)
{
const label curProc = cellToProc_[patchFaceCells[facei]];
// add the face without turning index
procFaceAddressing_[curProc].append(patchStart+facei+1);
// increment the number of faces for this patch
procPatchSize_[curProc][patchi]++;
}
}
else
{
const cyclicPolyPatch& pp = refCast<const cyclicPolyPatch>
(
patches[patchi]
);
// cyclic: check opposite side on this processor
const labelUList& patchFaceCells = pp.faceCells();
const labelUList& nbrPatchFaceCells =
pp.nbrPatch().faceCells();
forAll(patchFaceCells, facei)
{
const label curProc = cellToProc_[patchFaceCells[facei]];
const label nbrProc = cellToProc_[nbrPatchFaceCells[facei]];
if (curProc == nbrProc)
{
// add the face without turning index
procFaceAddressing_[curProc].append(patchStart+facei+1);
// increment the number of faces for this patch
procPatchSize_[curProc][patchi]++;
}
}
}
}
// Done internal bits of the new mesh and the ordinary patches.
// Per processor, from neighbour processor to the inter-processor patch
// that communicates with that neighbour
List<Map<label>> procNbrToInterPatch(nProcs_);
// Per processor the faces per inter-processor patch
List<DynamicList<DynamicList<label>>> interPatchFaces(nProcs_);
// Processor boundaries from internal faces
forAll(neighbour, facei)
{
label ownerProc = cellToProc_[owner[facei]];
label nbrProc = cellToProc_[neighbour[facei]];
if (ownerProc != nbrProc)
{
// inter - processor patch face found.
addInterProcFace
(
facei,
ownerProc,
nbrProc,
procNbrToInterPatch,
interPatchFaces
);
}
}
// Add the proper processor faces to the sub information. For faces
// originating from internal faces this is always -1.
List<labelListList> subPatchIDs(nProcs_);
List<labelListList> subPatchStarts(nProcs_);
forAll(interPatchFaces, proci)
{
label nInterfaces = interPatchFaces[proci].size();
subPatchIDs[proci].setSize(nInterfaces, labelList(1, label(-1)));
subPatchStarts[proci].setSize(nInterfaces, labelList(1, label(0)));
}
// Special handling needed for the case that multiple processor cyclic
// patches are created on each local processor domain, e.g. if a 3x3 case
// is decomposed using the decomposition:
//
// | 1 | 0 | 2 |
// cyclic left | 2 | 0 | 1 | cyclic right
// | 2 | 0 | 1 |
//
// - processors 1 and 2 will both have pieces of both cyclic left- and
// right sub-patches present
// - the interface patch faces are stored in a single list, where each
// sub-patch is referenced into the list using a patch start index and
// size
// - if the patches are in order (in the boundary file) of left, right
// - processor 1 will send: left, right
// - processor 1 will need to receive in reverse order: right, left
// - similarly for processor 2
// - the sub-patches are therefore generated in 4 passes of the patch lists
// 1. add faces from owner patch where local proc i < nbr proc i
// 2. add faces from nbr patch where local proc i < nbr proc i
// 3. add faces from owner patch where local proc i > nbr proc i
// 4. add faces from nbr patch where local proc i > nbr proc i
processInterCyclics
(
patches,
interPatchFaces,
procNbrToInterPatch,
subPatchIDs,
subPatchStarts,
true,
lessOp<label>()
);
processInterCyclics
(
patches,
interPatchFaces,
procNbrToInterPatch,
subPatchIDs,
subPatchStarts,
false,
lessOp<label>()
);
processInterCyclics
(
patches,
interPatchFaces,
procNbrToInterPatch,
subPatchIDs,
subPatchStarts,
false,
greaterOp<label>()
);
processInterCyclics
(
patches,
interPatchFaces,
procNbrToInterPatch,
subPatchIDs,
subPatchStarts,
true,
greaterOp<label>()
);
// Sort inter-proc patch by neighbour
labelList order;
forAll(procNbrToInterPatch, proci)
{
label nInterfaces = procNbrToInterPatch[proci].size();
procNeighbourProcessors_[proci].setSize(nInterfaces);
procProcessorPatchSize_[proci].setSize(nInterfaces);
procProcessorPatchStartIndex_[proci].setSize(nInterfaces);
procProcessorPatchSubPatchIDs_[proci].setSize(nInterfaces);
procProcessorPatchSubPatchStarts_[proci].setSize(nInterfaces);
// Info<< "Processor " << proci << endl;
// Get sorted neighbour processors
const Map<label>& curNbrToInterPatch = procNbrToInterPatch[proci];
labelList nbrs = curNbrToInterPatch.toc();
sortedOrder(nbrs, order);
DynamicList<DynamicList<label>>& curInterPatchFaces =
interPatchFaces[proci];
forAll(nbrs, i)
{
const label nbrProc = nbrs[i];
const label interPatch = curNbrToInterPatch[nbrProc];
procNeighbourProcessors_[proci][i] = nbrProc;
procProcessorPatchSize_[proci][i] =
curInterPatchFaces[interPatch].size();
procProcessorPatchStartIndex_[proci][i] =
procFaceAddressing_[proci].size();
// Add size as last element to substarts and transfer
append
(
subPatchStarts[proci][interPatch],
curInterPatchFaces[interPatch].size()
);
procProcessorPatchSubPatchIDs_[proci][i].transfer
(
subPatchIDs[proci][interPatch]
);
procProcessorPatchSubPatchStarts_[proci][i].transfer
(
subPatchStarts[proci][interPatch]
);
// Info<< " nbr:" << nbrProc << endl;
// Info<< " interpatch:" << interPatch << endl;
// Info<< " size:" << procProcessorPatchSize_[proci][i] << endl;
// Info<< " start:" << procProcessorPatchStartIndex_[proci][i]
// << endl;
// Info<< " subPatches:"
// << procProcessorPatchSubPatchIDs_[proci][i]
// << endl;
// Info<< " subStarts:"
// << procProcessorPatchSubPatchStarts_[proci][i] << endl;
// And add all the face labels for interPatch
DynamicList<label>& interPatchFaces =
curInterPatchFaces[interPatch];
forAll(interPatchFaces, j)
{
procFaceAddressing_[proci].append(interPatchFaces[j]);
}
interPatchFaces.clearStorage();
}
curInterPatchFaces.clearStorage();
procFaceAddressing_[proci].shrink();
}
////XXXXXXX
//// Print a bit
// forAll(procPatchStartIndex_, proci)
// {
// Info<< "Processor:" << proci << endl;
//
// Info<< " total faces:" << procFaceAddressing_[proci].size()
// << endl;
//
// const labelList& curProcPatchStartIndex = procPatchStartIndex_[proci];
//
// forAll(curProcPatchStartIndex, patchi)
// {
// Info<< " patch:" << patchi
// << "\tstart:" << curProcPatchStartIndex[patchi]
// << "\tsize:" << procPatchSize_[proci][patchi]
// << endl;
// }
// }
// Info<< endl;
//
// forAll(procNeighbourProcessors_, proci)
// {
// Info<< "Processor " << proci << endl;
//
// forAll(procNeighbourProcessors_[proci], i)
// {
// Info<< " nbr:" << procNeighbourProcessors_[proci][i] << endl;
// Info<< " size:" << procProcessorPatchSize_[proci][i] << endl;
// Info<< " start:" << procProcessorPatchStartIndex_[proci][i]
// << endl;
// }
// }
// Info<< endl;
//
// forAll(procFaceAddressing_, proci)
// {
// Info<< "Processor:" << proci << endl;
//
// Info<< " faces:" << procFaceAddressing_[proci] << endl;
// }
Info<< "\nDistributing points to processors" << endl;
// For every processor, loop through the list of faces for the processor.
// For every face, loop through the list of points and mark the point as
// used for the processor. Collect the list of used points for the
// processor.
forAll(procPointAddressing_, proci)
{
boolList pointLabels(nPoints(), false);
// Get reference to list of used faces
const labelList& procFaceLabels = procFaceAddressing_[proci];
forAll(procFaceLabels, facei)
{
// Because of the turning index, some labels may be negative
const labelList& facePoints = fcs[mag(procFaceLabels[facei]) - 1];
forAll(facePoints, pointi)
{
// Mark the point as used
pointLabels[facePoints[pointi]] = true;
}
}
// Collect the used points
labelList& procPointLabels = procPointAddressing_[proci];
procPointLabels.setSize(pointLabels.size());
label nUsedPoints = 0;
forAll(pointLabels, pointi)
{
if (pointLabels[pointi])
{
procPointLabels[nUsedPoints] = pointi;
nUsedPoints++;
}
}
// Reset the size of used points
procPointLabels.setSize(nUsedPoints);
}
}
// ************************************************************************* //

125
applications/utilities/parallelProcessing/decomposePar/domainDecompositionTemplates.C
View File

@ -1,125 +0,0 @@
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2014-2020 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
\*---------------------------------------------------------------------------*/
#include "cyclicPolyPatch.H"
// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
template<class BinaryOp>
void Foam::domainDecomposition::processInterCyclics
(
const polyBoundaryMesh& patches,
List<DynamicList<DynamicList<label>>>& interPatchFaces,
List<Map<label>>& procNbrToInterPatch,
List<labelListList>& subPatchIDs,
List<labelListList>& subPatchStarts,
bool owner,
BinaryOp bop
) const
{
// Processor boundaries from split cyclics
forAll(patches, patchi)
{
if (isA<cyclicPolyPatch>(patches[patchi]))
{
const cyclicPolyPatch& pp = refCast<const cyclicPolyPatch>
(
patches[patchi]
);
if (pp.owner() != owner)
{
continue;
}
// cyclic: check opposite side on this processor
const labelUList& patchFaceCells = pp.faceCells();
const labelUList& nbrPatchFaceCells =
pp.nbrPatch().faceCells();
// Store old sizes. Used to detect which inter-proc patches
// have been added to.
labelListList oldInterfaceSizes(nProcs_);
forAll(oldInterfaceSizes, proci)
{
labelList& curOldSizes = oldInterfaceSizes[proci];
curOldSizes.setSize(interPatchFaces[proci].size());
forAll(curOldSizes, interI)
{
curOldSizes[interI] =
interPatchFaces[proci][interI].size();
}
}
// Add faces with different owner and neighbour processors
forAll(patchFaceCells, facei)
{
const label ownerProc = cellToProc_[patchFaceCells[facei]];
const label nbrProc = cellToProc_[nbrPatchFaceCells[facei]];
if (bop(ownerProc, nbrProc))
{
// inter - processor patch face found.
addInterProcFace
(
pp.start()+facei,
ownerProc,
nbrProc,
procNbrToInterPatch,
interPatchFaces
);
}
}
// 1. Check if any faces added to existing interfaces
forAll(oldInterfaceSizes, proci)
{
const labelList& curOldSizes = oldInterfaceSizes[proci];
forAll(curOldSizes, interI)
{
label oldSz = curOldSizes[interI];
if (interPatchFaces[proci][interI].size() > oldSz)
{
// Added faces to this interface. Add an entry
append(subPatchIDs[proci][interI], patchi);
append(subPatchStarts[proci][interI], oldSz);
}
}
}
// 2. Any new interfaces
forAll(subPatchIDs, proci)
{
label nIntfcs = interPatchFaces[proci].size();
subPatchIDs[proci].setSize(nIntfcs, labelList(1, patchi));
subPatchStarts[proci].setSize(nIntfcs, labelList(1, label(0)));
}
}
}
}
// ************************************************************************* //

76
applications/utilities/parallelProcessing/decomposePar/fvFieldDecomposer.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2011-2021 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2022 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -123,56 +123,59 @@ Foam::fvFieldDecomposer::fvFieldDecomposer
const fvMesh& completeMesh,
const fvMesh& procMesh,
const labelList& faceAddressing,
const labelList& cellAddressing,
const labelList& boundaryAddressing
const labelList& cellAddressing
)
:
completeMesh_(completeMesh),
procMesh_(procMesh),
faceAddressing_(faceAddressing),
cellAddressing_(cellAddressing),
boundaryAddressing_(boundaryAddressing),
patchFieldDecomposerPtrs_
(
procMesh_.boundary().size(),
static_cast<patchFieldDecomposer*>(nullptr)
),
processorVolPatchFieldDecomposerPtrs_
(
procMesh_.boundary().size(),
static_cast<processorVolPatchFieldDecomposer*>(nullptr)
)
patchFieldDecomposers_(procMesh_.boundary().size()),
processorVolPatchFieldDecomposers_(procMesh_.boundary().size())
{
forAll(boundaryAddressing_, patchi)
forAll(procMesh_.boundary(), procPatchi)
{
const fvPatch& procPatch = procMesh.boundary()[patchi];
const fvPatch& procPatch = procMesh.boundary()[procPatchi];
label fromPatchi = boundaryAddressing_[patchi];
if (fromPatchi < 0 && isA<processorCyclicFvPatch>(procPatch))
// Determine the index of the corresponding complete patch
label completePatchi = -1;
if (procPatchi < completeMesh_.boundary().size())
{
const label referPatchi =
completePatchi = procPatchi;
}
else if (isA<processorCyclicFvPatch>(procPatch))
{
completePatchi =
refCast<const processorCyclicPolyPatch>
(procPatch.patch()).referPatchID();
fromPatchi = boundaryAddressing_[referPatchi];
}
if (fromPatchi >= 0)
// If there is a corresponding complete patch, then create a mapper
if (completePatchi >= 0)
{
patchFieldDecomposerPtrs_[patchi] = new patchFieldDecomposer
patchFieldDecomposers_.set
(
procMesh_.boundary()[patchi].patchSlice(faceAddressing_),
completeMesh_.boundaryMesh()[fromPatchi].start()
procPatchi,
new patchFieldDecomposer
(
procPatch.patchSlice(faceAddressing_),
completeMesh_.boundaryMesh()[completePatchi].start()
)
);
}
if (boundaryAddressing_[patchi] < 0)
// If this is a processor patch then create a processor mapper
if (procPatchi >= completeMesh_.boundary().size())
{
processorVolPatchFieldDecomposerPtrs_[patchi] =
processorVolPatchFieldDecomposers_.set
(
procPatchi,
new processorVolPatchFieldDecomposer
(
completeMesh_,
procMesh_.boundary()[patchi].patchSlice(faceAddressing_)
);
procPatch.patchSlice(faceAddressing_)
)
);
}
}
}
@ -181,22 +184,7 @@ Foam::fvFieldDecomposer::fvFieldDecomposer
// * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * //
Foam::fvFieldDecomposer::~fvFieldDecomposer()
{
forAll(patchFieldDecomposerPtrs_, patchi)
{
if (patchFieldDecomposerPtrs_[patchi])
{
delete patchFieldDecomposerPtrs_[patchi];
}
}
{}
forAll(processorVolPatchFieldDecomposerPtrs_, patchi)
{
if (processorVolPatchFieldDecomposerPtrs_[patchi])
{
delete processorVolPatchFieldDecomposerPtrs_[patchi];
}
}
}
// ************************************************************************* //

20
applications/utilities/parallelProcessing/decomposePar/fvFieldDecomposer.H
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2011-2021 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2022 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -53,9 +53,10 @@ class IOobjectList;
class fvFieldDecomposer
{
public:
// Public Classes
//- Patch field decomposer class
class patchFieldDecomposer
:
@ -83,7 +84,6 @@ public:
);
};
//- Processor patch field decomposer class. Maps either owner or
// neighbour data (no interpolate anymore - processorFvPatchField
// holds neighbour data)
@ -128,14 +128,12 @@ private:
//- Reference to cell addressing
const labelList& cellAddressing_;
//- Reference to boundary addressing
const labelList& boundaryAddressing_;
//- List of patch field decomposers
List<patchFieldDecomposer*> patchFieldDecomposerPtrs_;
PtrList<patchFieldDecomposer> patchFieldDecomposers_;
List<processorVolPatchFieldDecomposer*>
processorVolPatchFieldDecomposerPtrs_;
//- ...
PtrList<processorVolPatchFieldDecomposer>
processorVolPatchFieldDecomposers_;
// Private Member Functions
@ -160,8 +158,7 @@ public:
const fvMesh& completeMesh,
const fvMesh& procMesh,
const labelList& faceAddressing,
const labelList& cellAddressing,
const labelList& boundaryAddressing
const labelList& cellAddressing
);
//- Disallow default bitwise copy construction
@ -191,6 +188,7 @@ public:
const GeometricField<Type, fvsPatchField, surfaceMesh>& field
) const;
//- Decompose a list of fields
template<class GeoField>
void decomposeFields(const PtrList<GeoField>& fields) const;

77
applications/utilities/parallelProcessing/decomposePar/fvFieldDecomposerDecomposeFields.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2011-2021 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2022 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -75,17 +75,17 @@ Foam::fvFieldDecomposer::decomposeField
) const
{
// 1. Create the complete field with dummy patch fields
PtrList<fvPatchField<Type>> patchFields(boundaryAddressing_.size());
PtrList<fvPatchField<Type>> patchFields(procMesh_.boundary().size());
forAll(boundaryAddressing_, patchi)
forAll(procMesh_.boundary(), procPatchi)
{
patchFields.set
(
patchi,
procPatchi,
fvPatchField<Type>::New
(
calculatedFvPatchField<Type>::typeName,
procMesh_.boundary()[patchi],
procMesh_.boundary()[procPatchi],
DimensionedField<Type, volMesh>::null()
)
);
@ -120,33 +120,38 @@ Foam::fvFieldDecomposer::decomposeField
typename GeometricField<Type, fvPatchField, volMesh>::
Boundary& bf = resF.boundaryFieldRef();
forAll(bf, patchi)
forAll(bf, procPatchi)
{
const fvPatch& procPatch = procMesh_.boundary()[patchi];
const fvPatch& procPatch = procMesh_.boundary()[procPatchi];
label fromPatchi = boundaryAddressing_[patchi];
if (fromPatchi < 0 && isA<processorCyclicFvPatch>(procPatch))
// Determine the index of the corresponding complete patch
label completePatchi = -1;
if (procPatchi < completeMesh_.boundary().size())
{
completePatchi = procPatchi;
}
else if (isA<processorCyclicFvPatch>(procPatch))
{
const label referPatchi =
refCast<const processorCyclicPolyPatch>
(procPatch.patch()).referPatchID();
if (field.boundaryField()[referPatchi].overridesConstraint())
{
fromPatchi = boundaryAddressing_[referPatchi];
completePatchi = referPatchi;
}
}
if (fromPatchi >= 0)
if (completePatchi != -1)
{
bf.set
(
patchi,
procPatchi,
fvPatchField<Type>::New
(
field.boundaryField()[fromPatchi],
field.boundaryField()[completePatchi],
procPatch,
resF(),
*patchFieldDecomposerPtrs_[patchi]
patchFieldDecomposers_[procPatchi]
)
);
}
@ -154,12 +159,12 @@ Foam::fvFieldDecomposer::decomposeField
{
bf.set
(
patchi,
procPatchi,
new processorCyclicFvPatchField<Type>
(
procPatch,
resF(),
(*processorVolPatchFieldDecomposerPtrs_[patchi])
processorVolPatchFieldDecomposers_[procPatchi]
(
field.primitiveField()
)
@ -170,12 +175,12 @@ Foam::fvFieldDecomposer::decomposeField
{
bf.set
(
patchi,
procPatchi,
new processorFvPatchField<Type>
(
procPatch,
resF(),
(*processorVolPatchFieldDecomposerPtrs_[patchi])
processorVolPatchFieldDecomposers_[procPatchi]
(
field.primitiveField()
)
@ -186,7 +191,7 @@ Foam::fvFieldDecomposer::decomposeField
{
bf.set
(
patchi,
procPatchi,
new emptyFvPatchField<Type>
(
procPatch,
@ -213,10 +218,6 @@ Foam::fvFieldDecomposer::decomposeField
const GeometricField<Type, fvsPatchField, surfaceMesh>& field
) const
{
// Apply flipping to surfaceScalarFields only
const bool doFlip = (pTraits<Type>::nComponents == 1);
// Problem with addressing when a processor patch picks up both internal
// faces and faces from cyclic boundaries. This is a bit of a hack, but
// I cannot find a better solution without making the internal storage
@ -243,17 +244,17 @@ Foam::fvFieldDecomposer::decomposeField
// 1. Create the complete field with dummy patch fields
PtrList<fvsPatchField<Type>> patchFields(boundaryAddressing_.size());
PtrList<fvsPatchField<Type>> patchFields(procMesh_.boundary().size());
forAll(boundaryAddressing_, patchi)
forAll(procMesh_.boundary(), procPatchi)
{
patchFields.set
(
patchi,
procPatchi,
fvsPatchField<Type>::New
(
calculatedFvsPatchField<Type>::typeName,
procMesh_.boundary()[patchi],
procMesh_.boundary()[procPatchi],
DimensionedField<Type, surfaceMesh>::null()
)
);
@ -282,7 +283,7 @@ Foam::fvFieldDecomposer::decomposeField
faceAddressing_,
procMesh_.nInternalFaces()
),
doFlip
isFlux(field)
),
patchFields
)
@ -296,21 +297,21 @@ Foam::fvFieldDecomposer::decomposeField
typename GeometricField<Type, fvsPatchField, surfaceMesh>::
Boundary& bf = resF.boundaryFieldRef();
forAll(boundaryAddressing_, patchi)
forAll(procMesh_.boundary(), procPatchi)
{
const fvPatch& procPatch = procMesh_.boundary()[patchi];
const fvPatch& procPatch = procMesh_.boundary()[procPatchi];
if (boundaryAddressing_[patchi] >= 0)
if (procPatchi < completeMesh_.boundary().size())
{
bf.set
(
patchi,
procPatchi,
fvsPatchField<Type>::New
(
field.boundaryField()[boundaryAddressing_[patchi]],
field.boundaryField()[procPatchi],
procPatch,
resF(),
*patchFieldDecomposerPtrs_[patchi]
patchFieldDecomposers_[procPatchi]
)
);
}
@ -319,7 +320,7 @@ Foam::fvFieldDecomposer::decomposeField
// Do our own mapping. Avoids a lot of mapping complexity.
bf.set
(
patchi,
procPatchi,
new processorCyclicFvsPatchField<Type>
(
procPatch,
@ -328,7 +329,7 @@ Foam::fvFieldDecomposer::decomposeField
(
allFaceField,
procPatch.patchSlice(faceAddressing_),
doFlip
isFlux(field)
)
)
);
@ -338,7 +339,7 @@ Foam::fvFieldDecomposer::decomposeField
// Do our own mapping. Avoids a lot of mapping complexity.
bf.set
(
patchi,
procPatchi,
new processorFvsPatchField<Type>
(
procPatch,
@ -347,7 +348,7 @@ Foam::fvFieldDecomposer::decomposeField
(
allFaceField,
procPatch.patchSlice(faceAddressing_),
doFlip
isFlux(field)
)
)
);

40
applications/utilities/parallelProcessing/decomposePar/pointFieldDecomposer.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2011-2018 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2022 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -78,29 +78,27 @@ Foam::pointFieldDecomposer::pointFieldDecomposer
(
const pointMesh& completeMesh,
const pointMesh& procMesh,
const labelList& pointAddressing,
const labelList& boundaryAddressing
const labelList& pointAddressing
)
:
completeMesh_(completeMesh),
procMesh_(procMesh),
pointAddressing_(pointAddressing),
boundaryAddressing_(boundaryAddressing),
patchFieldDecomposerPtrs_
(
procMesh_.boundary().size(),
static_cast<patchFieldDecomposer*>(nullptr)
)
patchFieldDecomposers_(procMesh_.boundary().size())
{
forAll(boundaryAddressing_, patchi)
forAll(procMesh_.boundary(), patchi)
{
if (boundaryAddressing_[patchi] >= 0)
if (patchi < completeMesh_.boundary().size())
{
patchFieldDecomposerPtrs_[patchi] = new patchFieldDecomposer
patchFieldDecomposers_.set
(
completeMesh_.boundary()[boundaryAddressing_[patchi]],
procMesh_.boundary()[patchi],
pointAddressing_
patchi,
new patchFieldDecomposer
(
completeMesh_.boundary()[patchi],
procMesh_.boundary()[patchi],
pointAddressing_
)
);
}
}
@ -110,17 +108,7 @@ Foam::pointFieldDecomposer::pointFieldDecomposer
// * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * //
Foam::pointFieldDecomposer::~pointFieldDecomposer()
{
forAll(patchFieldDecomposerPtrs_, patchi)
{
if (patchFieldDecomposerPtrs_[patchi])
{
delete patchFieldDecomposerPtrs_[patchi];
}
}
}
{}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
// ************************************************************************* //

14
applications/utilities/parallelProcessing/decomposePar/pointFieldDecomposer.H
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2011-2020 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2022 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -51,9 +51,10 @@ namespace Foam
class pointFieldDecomposer
{
public:
// Public classes
//- Point patch field decomposer class
class patchFieldDecomposer
:
@ -116,11 +117,8 @@ private:
//- Reference to point addressing
const labelList& pointAddressing_;
//- Reference to boundary addressing
const labelList& boundaryAddressing_;
//- List of patch field decomposers
List<patchFieldDecomposer*> patchFieldDecomposerPtrs_;
PtrList<patchFieldDecomposer> patchFieldDecomposers_;
public:
@ -132,8 +130,7 @@ public:
(
const pointMesh& completeMesh,
const pointMesh& procMesh,
const labelList& pointAddressing,
const labelList& boundaryAddressing
const labelList& pointAddressing
);
//- Disallow default bitwise copy construction
@ -154,6 +151,7 @@ public:
const GeometricField<Type, pointPatchField, pointMesh>&
) const;
//- Decompose a list of fields
template<class GeoField>
void decomposeFields(const PtrList<GeoField>& fields) const;

12
applications/utilities/parallelProcessing/decomposePar/pointFieldDecomposerDecomposeFields.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2011-2018 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2022 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -39,22 +39,22 @@ Foam::pointFieldDecomposer::decomposeField
Field<Type> internalField(field.primitiveField(), pointAddressing_);
// Create a list of pointers for the patchFields
PtrList<pointPatchField<Type>> patchFields(boundaryAddressing_.size());
PtrList<pointPatchField<Type>> patchFields(procMesh_.boundary().size());
// Create and map the patch field values
forAll(boundaryAddressing_, patchi)
forAll(procMesh_.boundary(), patchi)
{
if (patchFieldDecomposerPtrs_[patchi])
if (patchi < completeMesh_.boundary().size())
{
patchFields.set
(
patchi,
pointPatchField<Type>::New
(
field.boundaryField()[boundaryAddressing_[patchi]],
field.boundaryField()[patchi],
procMesh_.boundary()[patchi],
DimensionedField<Type, pointMesh>::null(),
*patchFieldDecomposerPtrs_[patchi]
patchFieldDecomposers_[patchi]
)
);
}

40
applications/utilities/parallelProcessing/reconstructPar/reconstructPar.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2011-2021 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2022 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -306,16 +306,27 @@ int main(int argc, char *argv[])
// Check if any new meshes need to be read.
fvMesh::readUpdateState meshStat = mesh.readUpdate();
fvMesh::readUpdateState procStat = procMeshes.readUpdate();
if
(
meshStat != procStat
if (procStat == fvMesh::POINTS_MOVED)
{
// Reconstruct the points for moving mesh cases and write
// them out
procMeshes.reconstructPoints(mesh);
}
else if (meshStat != procStat)
// Allowed status difference. Mesh has moved in parallel, but
// the change has not yet been reconstructed.
&& !(
meshStat == fvMesh::UNCHANGED
&& procStat == fvMesh::POINTS_MOVED
)
// Allowed status difference. Mesh has changed topology in both
// cases, and this has lead to new processor interfaces
// (probably). These interfaces are compatible with a
// reconstructed mesh in which patches have not changed.
&& !(
meshStat == fvMesh::TOPO_CHANGE
&& procStat == fvMesh::TOPO_PATCH_CHANGE
)
)
{
WarningInFunction
<< "readUpdate for the reconstructed mesh:"
@ -328,6 +339,11 @@ int main(int argc, char *argv[])
<< "mesh directories." << endl;
}
// Reconstruct and write the points for moving mesh cases
if (procStat == fvMesh::POINTS_MOVED)
{
procMeshes.reconstructPoints(mesh);
}
// Get list of objects from processor0 database
IOobjectList objects
@ -346,8 +362,7 @@ int main(int argc, char *argv[])
mesh,
procMeshes.meshes(),
procMeshes.faceProcAddressing(),
procMeshes.cellProcAddressing(),
procMeshes.boundaryProcAddressing()
procMeshes.cellProcAddressing()
);
fvReconstructor.reconstructFvVolumeInternalFields<scalar>
@ -455,8 +470,7 @@ int main(int argc, char *argv[])
(
pMesh,
pMeshes,
procMeshes.pointProcAddressing(),
procMeshes.boundaryProcAddressing()
procMeshes.pointProcAddressing()
);
pointReconstructor.reconstructFields<scalar>

45
applications/utilities/parallelProcessing/reconstructParMesh/reconstructParMesh.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2011-2021 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2022 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -343,7 +343,6 @@ int main(int argc, char *argv[])
labelListList cellProcAddressing(nProcs);
labelListList faceProcAddressing(nProcs);
labelListList pointProcAddressing(nProcs);
labelListList boundaryProcAddressing(nProcs);
// Internal faces on the final reconstructed mesh
label masterInternalFaces;
@ -391,8 +390,6 @@ int main(int argc, char *argv[])
cellProcAddressing[proci] = identity(meshToAdd.nCells());
faceProcAddressing[proci] = identity(meshToAdd.nFaces());
pointProcAddressing[proci] = identity(meshToAdd.nPoints());
boundaryProcAddressing[proci] =
identity(meshToAdd.boundaryMesh().size());
// Find shared points/faces
autoPtr<faceCoupleInfo> couples = determineCoupledFaces
@ -429,11 +426,6 @@ int main(int argc, char *argv[])
map().addedPointMap(),
pointProcAddressing[proci]
);
inplaceRenumber
(
map().addedPatchMap(),
boundaryProcAddressing[proci]
);
}
// Merge the meshes
@ -487,11 +479,6 @@ int main(int argc, char *argv[])
map().oldPointMap(),
pointProcAddressing[mergedI]
);
inplaceRenumber
(
map().oldPatchMap(),
boundaryProcAddressing[mergedI]
);
}
// Added processor
@ -517,11 +504,6 @@ int main(int argc, char *argv[])
map().addedPointMap(),
pointProcAddressing[addedI]
);
inplaceRenumber
(
map().addedPatchMap(),
boundaryProcAddressing[addedI]
);
}
masterMesh.set(next, nullptr);
@ -696,31 +678,6 @@ int main(int argc, char *argv[])
cellProcAddressing[proci]
).write();
// From processor patch to reconstructed mesh patch
Info<< "Writing boundaryProcAddressing to "
<< databases[proci].caseName()
/procMesh.facesInstance()
/polyMesh::meshSubDir
<< endl;
labelIOList
(
IOobject
(
"boundaryProcAddressing",
procMesh.facesInstance(),
polyMesh::meshSubDir,
procMesh,
IOobject::NO_READ,
IOobject::NO_WRITE,
false // Do not register
),
boundaryProcAddressing[proci]
).write();
Info<< endl;
}
}