mirror of
https://develop.openfoam.com/Development/openfoam.git
synced 2025-11-28 03:28:01 +00:00
ENH: parallel: overhaul of parallel mapping
- redistributePar to have almost (complete) functionality of decomposePar+reconstructPar - low-level distributed Field mapping - support for mapping surfaceFields (including flipping faces) - support for decomposing/reconstructing refinement data
This commit is contained in:
mattijs
@ -3,7 +3,7 @@
|
||||
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
|
||||
\\ / O peration |
|
||||
\\ / A nd | Copyright (C) 2011-2014 OpenFOAM Foundation
|
||||
\\/ M anipulation |
|
||||
\\/ M anipulation | Copyright (C) 2015 OpenCFD Ltd.
|
||||
-------------------------------------------------------------------------------
|
||||
License
|
||||
This file is part of OpenFOAM.
|
||||
@ -36,6 +36,11 @@ InClass
|
||||
#include "minData.H"
|
||||
#include "FaceCellWave.H"
|
||||
|
||||
#include "preserveBafflesConstraint.H"
|
||||
#include "preservePatchesConstraint.H"
|
||||
#include "preserveFaceZonesConstraint.H"
|
||||
#include "singleProcessorFaceSetsConstraint.H"
|
||||
|
||||
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
|
||||
|
||||
namespace Foam
|
||||
@ -44,6 +49,130 @@ namespace Foam
|
||||
defineRunTimeSelectionTable(decompositionMethod, dictionary);
|
||||
}
|
||||
|
||||
// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
|
||||
|
||||
Foam::decompositionMethod::decompositionMethod
|
||||
(
|
||||
const dictionary& decompositionDict
|
||||
)
|
||||
:
|
||||
decompositionDict_(decompositionDict),
|
||||
nProcessors_
|
||||
(
|
||||
readLabel(decompositionDict.lookup("numberOfSubdomains"))
|
||||
)
|
||||
{
|
||||
// Read any constraints
|
||||
wordList constraintTypes_;
|
||||
if (decompositionDict_.found("constraints"))
|
||||
{
|
||||
//PtrList<dictionary> constraintsList
|
||||
//(
|
||||
// decompositionDict_.lookup("constraints")
|
||||
//);
|
||||
//forAll(constraintsList, i)
|
||||
//{
|
||||
// const dictionary& dict = constraintsList[i];
|
||||
const dictionary& constraintsList = decompositionDict_.subDict
|
||||
(
|
||||
"constraints"
|
||||
);
|
||||
forAllConstIter(dictionary, constraintsList, iter)
|
||||
{
|
||||
const dictionary& dict = iter().dict();
|
||||
|
||||
constraintTypes_.append(dict.lookup("type"));
|
||||
|
||||
constraints_.append
|
||||
(
|
||||
decompositionConstraint::New
|
||||
(
|
||||
dict,
|
||||
constraintTypes_.last()
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Backwards compatibility
|
||||
if
|
||||
(
|
||||
decompositionDict_.found("preserveBaffles")
|
||||
&& findIndex
|
||||
(
|
||||
constraintTypes_,
|
||||
decompositionConstraints::preserveBafflesConstraint::typeName
|
||||
) == -1
|
||||
)
|
||||
{
|
||||
constraints_.append
|
||||
(
|
||||
new decompositionConstraints::preserveBafflesConstraint()
|
||||
);
|
||||
}
|
||||
|
||||
if
|
||||
(
|
||||
decompositionDict_.found("preservePatches")
|
||||
&& findIndex
|
||||
(
|
||||
constraintTypes_,
|
||||
decompositionConstraints::preservePatchesConstraint::typeName
|
||||
) == -1
|
||||
)
|
||||
{
|
||||
const wordReList pNames(decompositionDict_.lookup("preservePatches"));
|
||||
|
||||
constraints_.append
|
||||
(
|
||||
new decompositionConstraints::preservePatchesConstraint(pNames)
|
||||
);
|
||||
}
|
||||
|
||||
if
|
||||
(
|
||||
decompositionDict_.found("preserveFaceZones")
|
||||
&& findIndex
|
||||
(
|
||||
constraintTypes_,
|
||||
decompositionConstraints::preserveFaceZonesConstraint::typeName
|
||||
) == -1
|
||||
)
|
||||
{
|
||||
const wordReList zNames(decompositionDict_.lookup("preserveFaceZones"));
|
||||
|
||||
constraints_.append
|
||||
(
|
||||
new decompositionConstraints::preserveFaceZonesConstraint(zNames)
|
||||
);
|
||||
}
|
||||
|
||||
if
|
||||
(
|
||||
decompositionDict_.found("singleProcessorFaceSets")
|
||||
&& findIndex
|
||||
(
|
||||
constraintTypes_,
|
||||
decompositionConstraints::preserveFaceZonesConstraint::typeName
|
||||
) == -1
|
||||
)
|
||||
{
|
||||
const List<Tuple2<word, label> > zNameAndProcs
|
||||
(
|
||||
decompositionDict_.lookup("singleProcessorFaceSets")
|
||||
);
|
||||
|
||||
constraints_.append
|
||||
(
|
||||
new decompositionConstraints::singleProcessorFaceSetsConstraint
|
||||
(
|
||||
zNameAndProcs
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
|
||||
|
||||
Foam::autoPtr<Foam::decompositionMethod> Foam::decompositionMethod::New
|
||||
@ -53,12 +182,6 @@ Foam::autoPtr<Foam::decompositionMethod> Foam::decompositionMethod::New
|
||||
{
|
||||
word methodType(decompositionDict.lookup("method"));
|
||||
|
||||
if (methodType == "scotch" && Pstream::parRun())
|
||||
{
|
||||
methodType = "ptscotch";
|
||||
}
|
||||
|
||||
|
||||
Info<< "Selecting decompositionMethod " << methodType << endl;
|
||||
|
||||
dictionaryConstructorTable::iterator cstrIter =
|
||||
@ -370,6 +493,210 @@ void Foam::decompositionMethod::calcCellCells
|
||||
}
|
||||
|
||||
|
||||
void Foam::decompositionMethod::calcCellCells
|
||||
(
|
||||
const polyMesh& mesh,
|
||||
const labelList& agglom,
|
||||
const label nLocalCoarse,
|
||||
const bool parallel,
|
||||
CompactListList<label>& cellCells,
|
||||
CompactListList<scalar>& cellCellWeights
|
||||
)
|
||||
{
|
||||
const labelList& faceOwner = mesh.faceOwner();
|
||||
const labelList& faceNeighbour = mesh.faceNeighbour();
|
||||
const polyBoundaryMesh& patches = mesh.boundaryMesh();
|
||||
|
||||
|
||||
// Create global cell numbers
|
||||
// ~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
globalIndex globalAgglom
|
||||
(
|
||||
nLocalCoarse,
|
||||
Pstream::msgType(),
|
||||
Pstream::worldComm,
|
||||
parallel
|
||||
);
|
||||
|
||||
|
||||
// Get agglomerate owner on other side of coupled faces
|
||||
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
labelList globalNeighbour(mesh.nFaces()-mesh.nInternalFaces());
|
||||
|
||||
forAll(patches, patchI)
|
||||
{
|
||||
const polyPatch& pp = patches[patchI];
|
||||
|
||||
if (pp.coupled() && (parallel || !isA<processorPolyPatch>(pp)))
|
||||
{
|
||||
label faceI = pp.start();
|
||||
label bFaceI = pp.start() - mesh.nInternalFaces();
|
||||
|
||||
forAll(pp, i)
|
||||
{
|
||||
globalNeighbour[bFaceI] = globalAgglom.toGlobal
|
||||
(
|
||||
agglom[faceOwner[faceI]]
|
||||
);
|
||||
|
||||
bFaceI++;
|
||||
faceI++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Get the cell on the other side of coupled patches
|
||||
syncTools::swapBoundaryFaceList(mesh, globalNeighbour);
|
||||
|
||||
|
||||
// Count number of faces (internal + coupled)
|
||||
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
// Number of faces per coarse cell
|
||||
labelList nFacesPerCell(nLocalCoarse, 0);
|
||||
|
||||
for (label faceI = 0; faceI < mesh.nInternalFaces(); faceI++)
|
||||
{
|
||||
label own = agglom[faceOwner[faceI]];
|
||||
label nei = agglom[faceNeighbour[faceI]];
|
||||
|
||||
nFacesPerCell[own]++;
|
||||
nFacesPerCell[nei]++;
|
||||
}
|
||||
|
||||
forAll(patches, patchI)
|
||||
{
|
||||
const polyPatch& pp = patches[patchI];
|
||||
|
||||
if (pp.coupled() && (parallel || !isA<processorPolyPatch>(pp)))
|
||||
{
|
||||
label faceI = pp.start();
|
||||
label bFaceI = pp.start()-mesh.nInternalFaces();
|
||||
|
||||
forAll(pp, i)
|
||||
{
|
||||
label own = agglom[faceOwner[faceI]];
|
||||
|
||||
label globalNei = globalNeighbour[bFaceI];
|
||||
if
|
||||
(
|
||||
!globalAgglom.isLocal(globalNei)
|
||||
|| globalAgglom.toLocal(globalNei) != own
|
||||
)
|
||||
{
|
||||
nFacesPerCell[own]++;
|
||||
}
|
||||
|
||||
faceI++;
|
||||
bFaceI++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Fill in offset and data
|
||||
// ~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
cellCells.setSize(nFacesPerCell);
|
||||
cellCellWeights.setSize(nFacesPerCell);
|
||||
|
||||
nFacesPerCell = 0;
|
||||
|
||||
labelList& m = cellCells.m();
|
||||
scalarList& w = cellCellWeights.m();
|
||||
const labelList& offsets = cellCells.offsets();
|
||||
|
||||
// For internal faces is just offsetted owner and neighbour
|
||||
for (label faceI = 0; faceI < mesh.nInternalFaces(); faceI++)
|
||||
{
|
||||
label own = agglom[faceOwner[faceI]];
|
||||
label nei = agglom[faceNeighbour[faceI]];
|
||||
|
||||
label ownIndex = offsets[own] + nFacesPerCell[own]++;
|
||||
label neiIndex = offsets[nei] + nFacesPerCell[nei]++;
|
||||
|
||||
m[ownIndex] = globalAgglom.toGlobal(nei);
|
||||
w[ownIndex] = mag(mesh.faceAreas()[faceI]);
|
||||
m[neiIndex] = globalAgglom.toGlobal(own);
|
||||
w[ownIndex] = mag(mesh.faceAreas()[faceI]);
|
||||
}
|
||||
|
||||
// For boundary faces is offsetted coupled neighbour
|
||||
forAll(patches, patchI)
|
||||
{
|
||||
const polyPatch& pp = patches[patchI];
|
||||
|
||||
if (pp.coupled() && (parallel || !isA<processorPolyPatch>(pp)))
|
||||
{
|
||||
label faceI = pp.start();
|
||||
label bFaceI = pp.start()-mesh.nInternalFaces();
|
||||
|
||||
forAll(pp, i)
|
||||
{
|
||||
label own = agglom[faceOwner[faceI]];
|
||||
|
||||
label globalNei = globalNeighbour[bFaceI];
|
||||
|
||||
if
|
||||
(
|
||||
!globalAgglom.isLocal(globalNei)
|
||||
|| globalAgglom.toLocal(globalNei) != own
|
||||
)
|
||||
{
|
||||
label ownIndex = offsets[own] + nFacesPerCell[own]++;
|
||||
m[ownIndex] = globalNei;
|
||||
w[ownIndex] = mag(mesh.faceAreas()[faceI]);
|
||||
}
|
||||
|
||||
faceI++;
|
||||
bFaceI++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Check for duplicates connections between cells
|
||||
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
// Done as postprocessing step since we now have cellCells.
|
||||
label newIndex = 0;
|
||||
labelHashSet nbrCells;
|
||||
|
||||
|
||||
if (cellCells.size() == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
label startIndex = cellCells.offsets()[0];
|
||||
|
||||
forAll(cellCells, cellI)
|
||||
{
|
||||
nbrCells.clear();
|
||||
nbrCells.insert(globalAgglom.toGlobal(cellI));
|
||||
|
||||
label endIndex = cellCells.offsets()[cellI+1];
|
||||
|
||||
for (label i = startIndex; i < endIndex; i++)
|
||||
{
|
||||
if (nbrCells.insert(cellCells.m()[i]))
|
||||
{
|
||||
cellCells.m()[newIndex] = cellCells.m()[i];
|
||||
cellCellWeights.m()[newIndex] = cellCellWeights.m()[i];
|
||||
newIndex++;
|
||||
}
|
||||
}
|
||||
startIndex = endIndex;
|
||||
cellCells.offsets()[cellI+1] = newIndex;
|
||||
cellCellWeights.offsets()[cellI+1] = newIndex;
|
||||
}
|
||||
|
||||
cellCells.m().setSize(newIndex);
|
||||
cellCellWeights.m().setSize(newIndex);
|
||||
}
|
||||
|
||||
|
||||
//void Foam::decompositionMethod::calcCellCells
|
||||
//(
|
||||
// const polyMesh& mesh,
|
||||
@ -1123,172 +1450,45 @@ void Foam::decompositionMethod::setConstraints
|
||||
{
|
||||
blockedFace.setSize(mesh.nFaces());
|
||||
blockedFace = true;
|
||||
//label nUnblocked = 0;
|
||||
|
||||
specifiedProcessorFaces.clear();
|
||||
explicitConnections.clear();
|
||||
|
||||
|
||||
if (decompositionDict_.found("preservePatches"))
|
||||
forAll(constraints_, constraintI)
|
||||
{
|
||||
wordList pNames(decompositionDict_.lookup("preservePatches"));
|
||||
|
||||
Info<< nl
|
||||
<< "Keeping owner of faces in patches " << pNames
|
||||
<< " on same processor. This only makes sense for cyclics." << endl;
|
||||
|
||||
const polyBoundaryMesh& patches = mesh.boundaryMesh();
|
||||
|
||||
forAll(pNames, i)
|
||||
{
|
||||
const label patchI = patches.findPatchID(pNames[i]);
|
||||
|
||||
if (patchI == -1)
|
||||
{
|
||||
FatalErrorIn("decompositionMethod::decompose(const polyMesh&)")
|
||||
<< "Unknown preservePatch " << pNames[i]
|
||||
<< endl << "Valid patches are " << patches.names()
|
||||
<< exit(FatalError);
|
||||
}
|
||||
|
||||
const polyPatch& pp = patches[patchI];
|
||||
|
||||
forAll(pp, i)
|
||||
{
|
||||
if (blockedFace[pp.start() + i])
|
||||
{
|
||||
blockedFace[pp.start() + i] = false;
|
||||
//nUnblocked++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (decompositionDict_.found("preserveFaceZones"))
|
||||
{
|
||||
wordList zNames(decompositionDict_.lookup("preserveFaceZones"));
|
||||
|
||||
Info<< nl
|
||||
<< "Keeping owner and neighbour of faces in zones " << zNames
|
||||
<< " on same processor" << endl;
|
||||
|
||||
const faceZoneMesh& fZones = mesh.faceZones();
|
||||
|
||||
forAll(zNames, i)
|
||||
{
|
||||
label zoneI = fZones.findZoneID(zNames[i]);
|
||||
|
||||
if (zoneI == -1)
|
||||
{
|
||||
FatalErrorIn("decompositionMethod::decompose(const polyMesh&)")
|
||||
<< "Unknown preserveFaceZone " << zNames[i]
|
||||
<< endl << "Valid faceZones are " << fZones.names()
|
||||
<< exit(FatalError);
|
||||
}
|
||||
|
||||
const faceZone& fz = fZones[zoneI];
|
||||
|
||||
forAll(fz, i)
|
||||
{
|
||||
if (blockedFace[fz[i]])
|
||||
{
|
||||
blockedFace[fz[i]] = false;
|
||||
//nUnblocked++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool preserveBaffles = decompositionDict_.lookupOrDefault
|
||||
(
|
||||
"preserveBaffles",
|
||||
false
|
||||
);
|
||||
if (preserveBaffles)
|
||||
{
|
||||
Info<< nl
|
||||
<< "Keeping owner of faces in baffles "
|
||||
<< " on same processor." << endl;
|
||||
|
||||
explicitConnections = localPointRegion::findDuplicateFacePairs(mesh);
|
||||
forAll(explicitConnections, i)
|
||||
{
|
||||
blockedFace[explicitConnections[i].first()] = false;
|
||||
blockedFace[explicitConnections[i].second()] = false;
|
||||
}
|
||||
}
|
||||
|
||||
if
|
||||
(
|
||||
decompositionDict_.found("preservePatches")
|
||||
|| decompositionDict_.found("preserveFaceZones")
|
||||
|| preserveBaffles
|
||||
)
|
||||
{
|
||||
syncTools::syncFaceList(mesh, blockedFace, andEqOp<bool>());
|
||||
//reduce(nUnblocked, sumOp<label>());
|
||||
}
|
||||
|
||||
|
||||
|
||||
// Specified processor for group of cells connected to faces
|
||||
|
||||
label nProcSets = 0;
|
||||
if (decompositionDict_.found("singleProcessorFaceSets"))
|
||||
{
|
||||
List<Tuple2<word, label> > zNameAndProcs
|
||||
constraints_[constraintI].add
|
||||
(
|
||||
decompositionDict_.lookup("singleProcessorFaceSets")
|
||||
mesh,
|
||||
blockedFace,
|
||||
specifiedProcessorFaces,
|
||||
specifiedProcessor,
|
||||
explicitConnections
|
||||
);
|
||||
|
||||
specifiedProcessorFaces.setSize(zNameAndProcs.size());
|
||||
specifiedProcessor.setSize(zNameAndProcs.size());
|
||||
|
||||
forAll(zNameAndProcs, setI)
|
||||
{
|
||||
Info<< "Keeping all cells connected to faceSet "
|
||||
<< zNameAndProcs[setI].first()
|
||||
<< " on processor " << zNameAndProcs[setI].second() << endl;
|
||||
|
||||
// Read faceSet
|
||||
faceSet fz(mesh, zNameAndProcs[setI].first());
|
||||
|
||||
specifiedProcessorFaces.set(setI, new labelList(fz.sortedToc()));
|
||||
specifiedProcessor[setI] = zNameAndProcs[setI].second();
|
||||
nProcSets += fz.size();
|
||||
}
|
||||
reduce(nProcSets, sumOp<label>());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Unblock all point connected faces
|
||||
// 1. Mark all points on specifiedProcessorFaces
|
||||
boolList procFacePoint(mesh.nPoints(), false);
|
||||
forAll(specifiedProcessorFaces, setI)
|
||||
{
|
||||
const labelList& set = specifiedProcessorFaces[setI];
|
||||
forAll(set, fI)
|
||||
{
|
||||
const face& f = mesh.faces()[set[fI]];
|
||||
forAll(f, fp)
|
||||
{
|
||||
procFacePoint[f[fp]] = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
syncTools::syncPointList(mesh, procFacePoint, orEqOp<bool>(), false);
|
||||
|
||||
// 2. Unblock all faces on procFacePoint
|
||||
forAll(procFacePoint, pointI)
|
||||
{
|
||||
if (procFacePoint[pointI])
|
||||
{
|
||||
const labelList& pFaces = mesh.pointFaces()[pointI];
|
||||
forAll(pFaces, i)
|
||||
{
|
||||
blockedFace[pFaces[i]] = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
syncTools::syncFaceList(mesh, blockedFace, andEqOp<bool>());
|
||||
void Foam::decompositionMethod::applyConstraints
|
||||
(
|
||||
const polyMesh& mesh,
|
||||
const boolList& blockedFace,
|
||||
const PtrList<labelList>& specifiedProcessorFaces,
|
||||
const labelList& specifiedProcessor,
|
||||
const List<labelPair>& explicitConnections,
|
||||
labelList& decomposition
|
||||
)
|
||||
{
|
||||
forAll(constraints_, constraintI)
|
||||
{
|
||||
constraints_[constraintI].apply
|
||||
(
|
||||
mesh,
|
||||
blockedFace,
|
||||
specifiedProcessorFaces,
|
||||
specifiedProcessor,
|
||||
explicitConnections,
|
||||
decomposition
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1299,6 +1499,8 @@ Foam::labelList Foam::decompositionMethod::decompose
|
||||
const scalarField& cellWeights
|
||||
)
|
||||
{
|
||||
// Collect all constraints
|
||||
|
||||
boolList blockedFace;
|
||||
PtrList<labelList> specifiedProcessorFaces;
|
||||
labelList specifiedProcessor;
|
||||
@ -1326,6 +1528,19 @@ Foam::labelList Foam::decompositionMethod::decompose
|
||||
explicitConnections // baffles
|
||||
);
|
||||
|
||||
|
||||
// Give any constraint the option of modifying the decomposition
|
||||
|
||||
applyConstraints
|
||||
(
|
||||
mesh,
|
||||
blockedFace,
|
||||
specifiedProcessorFaces,
|
||||
specifiedProcessor,
|
||||
explicitConnections,
|
||||
finalDecomp
|
||||
);
|
||||
|
||||
return finalDecomp;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user