zhyang-dev/OpenFOAM-6
1
0
Fork 0
mirror of https://github.com/OpenFOAM/OpenFOAM-6.git synced 2025-12-08 06:57:46 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

reconstructParMesh: Use tree method to improve speed and scaling

Browse Source 
Based on development contributed by Paul Edwards, Intel.
This commit is contained in:
Henry Weller
2017-05-09 09:32:25 +01:00
parent 130b051b54
commit 8f2735ebba
1 changed files with 189 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

202
applications/utilities/parallelProcessing/reconstructParMesh/reconstructParMesh.C
View File

@ -2,7 +2,7 @@
========= | ========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | \\ / O peration |
\\ / A nd | Copyright (C) 2011-2016 OpenFOAM Foundation \\ / A nd | Copyright (C) 2011-2017 OpenFOAM Foundation
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
@ -83,8 +83,11 @@ static void renumber
autoPtr<faceCoupleInfo> determineCoupledFaces autoPtr<faceCoupleInfo> determineCoupledFaces
( (
const bool fullMatch, const bool fullMatch,
const label proci, const label masterMeshProcStart,
const label masterMeshProcEnd,
const polyMesh& masterMesh, const polyMesh& masterMesh,
const label meshToAddProcStart,
const label meshToAddProcEnd,
const polyMesh& meshToAdd, const polyMesh& meshToAdd,
const scalar mergeDist const scalar mergeDist
) )
@ -109,7 +112,6 @@ autoPtr<faceCoupleInfo> determineCoupledFaces
const polyBoundaryMesh& masterPatches = masterMesh.boundaryMesh(); const polyBoundaryMesh& masterPatches = masterMesh.boundaryMesh();
const string toProcString("to" + name(proci));
DynamicList<label> masterFaces DynamicList<label> masterFaces
( (
@ -117,24 +119,35 @@ autoPtr<faceCoupleInfo> determineCoupledFaces
- masterMesh.nInternalFaces() - masterMesh.nInternalFaces()
); );
forAll(masterPatches, patchi) forAll(masterPatches, patchi)
{ {
const polyPatch& pp = masterPatches[patchi]; const polyPatch& pp = masterPatches[patchi];
if if (isA<processorPolyPatch>(pp))
{
for
( (
isA<processorPolyPatch>(pp) label proci=meshToAddProcStart;
&& ( proci<meshToAddProcEnd;
proci++
)
{
const string toProcString("to" + name(proci));
if (
pp.name().rfind(toProcString) pp.name().rfind(toProcString)
== (pp.name().size()-toProcString.size()) == (pp.name().size()-toProcString.size())
) )
)
{ {
label meshFacei = pp.start(); label meshFacei = pp.start();
forAll(pp, i) forAll(pp, i)
{ {
masterFaces.append(meshFacei++); masterFaces.append(meshFacei++);
} }
break;
}
}
} }
} }
masterFaces.shrink(); masterFaces.shrink();
@ -159,7 +172,19 @@ autoPtr<faceCoupleInfo> determineCoupledFaces
{ {
bool isConnected = false; bool isConnected = false;
for (label mergedProci = 0; mergedProci < proci; mergedProci++) for
(
label mergedProci=masterMeshProcStart;
!isConnected && (mergedProci < masterMeshProcEnd);
mergedProci++
)
{
for
(
label proci = meshToAddProcStart;
proci < meshToAddProcEnd;
proci++
)
{ {
const word fromProcString const word fromProcString
( (
@ -172,6 +197,7 @@ autoPtr<faceCoupleInfo> determineCoupledFaces
break; break;
} }
} }
}
if (isConnected) if (isConnected)
{ {
@ -622,8 +648,15 @@ int main(int argc, char *argv[])
{ {
// Construct empty mesh. // Construct empty mesh.
Info<< "Constructing empty mesh to add to." << nl << endl; // fvMesh** masterMesh = new fvMesh*[nProcs];
fvMesh masterMesh PtrList<fvMesh> masterMesh(nProcs);
for (label proci=0; proci<nProcs; proci++)
{
masterMesh.set
(
proci,
new fvMesh
( (
IOobject IOobject
( (
@ -635,15 +668,9 @@ int main(int argc, char *argv[])
xferCopy(pointField()), xferCopy(pointField()),
xferCopy(faceList()), xferCopy(faceList()),
xferCopy(cellList()) xferCopy(cellList())
)
); );
for (label proci = 0; proci < nProcs; proci++)
{
Info<< "Reading mesh to add from "
<< databases[proci].caseName()
<< " for time = " << databases[proci].timeName()
<< nl << endl;
fvMesh meshToAdd fvMesh meshToAdd
( (
IOobject IOobject
@ -661,37 +688,87 @@ int main(int argc, char *argv[])
boundaryProcAddressing[proci] = boundaryProcAddressing[proci] =
identity(meshToAdd.boundaryMesh().size()); identity(meshToAdd.boundaryMesh().size());
// Find geometrically shared points/faces.
autoPtr<faceCoupleInfo> couples = determineCoupledFaces
(
fullMatch,
proci,
proci,
masterMesh[proci],
proci,
proci,
meshToAdd,
mergeDist
);
// Add elements to mesh
autoPtr<mapAddedPolyMesh> map = fvMeshAdder::add
(
masterMesh[proci],
meshToAdd,
couples
);
// Added processor
renumber(map().addedCellMap(), cellProcAddressing[proci]);
renumber(map().addedFaceMap(), faceProcAddressing[proci]);
renumber(map().addedPointMap(), pointProcAddressing[proci]);
renumber(map().addedPatchMap(), boundaryProcAddressing[proci]);
}
for (label step=2; step<nProcs*2; step*=2)
{
for (label proci=0; proci<nProcs; proci+=step)
{
label next = proci + step/2;
if(next >= nProcs)
{
continue;
}
Info<< "Merging mesh " << proci << " with " << next << endl;
// Find geometrically shared points/faces. // Find geometrically shared points/faces.
autoPtr<faceCoupleInfo> couples = determineCoupledFaces autoPtr<faceCoupleInfo> couples = determineCoupledFaces
( (
fullMatch, fullMatch,
proci, proci,
masterMesh, next,
meshToAdd, masterMesh[proci],
next,
proci+step,
masterMesh[next],
mergeDist mergeDist
); );
// Add elements to mesh // Add elements to mesh
Info<< "Adding to master mesh" << nl << endl;
autoPtr<mapAddedPolyMesh> map = fvMeshAdder::add autoPtr<mapAddedPolyMesh> map = fvMeshAdder::add
( (
masterMesh, masterMesh[proci],
meshToAdd, masterMesh[next],
couples couples
); );
// Update all addressing so xxProcAddressing points to correct
// item in masterMesh.
// Processors that were already in masterMesh // Processors that were already in masterMesh
for (label mergedI = 0; mergedI < proci; mergedI++) for (label mergedI=proci; mergedI<next; mergedI++)
{ {
renumber(map().oldCellMap(), cellProcAddressing[mergedI]); renumber
renumber(map().oldFaceMap(), faceProcAddressing[mergedI]); (
renumber(map().oldPointMap(), pointProcAddressing[mergedI]); map().oldCellMap(),
cellProcAddressing[mergedI]
);
renumber
(
map().oldFaceMap(),
faceProcAddressing[mergedI]
);
renumber
(
map().oldPointMap(),
pointProcAddressing[mergedI]
);
// Note: boundary is special since can contain -1. // Note: boundary is special since can contain -1.
renumber renumber
( (
@ -701,28 +778,64 @@ int main(int argc, char *argv[])
} }
// Added processor // Added processor
renumber(map().addedCellMap(), cellProcAddressing[proci]); for
renumber(map().addedFaceMap(), faceProcAddressing[proci]); (
renumber(map().addedPointMap(), pointProcAddressing[proci]); label addedI=next;
renumber(map().addedPatchMap(), boundaryProcAddressing[proci]); addedI<min(proci+step, nProcs);
addedI++
)
{
renumber
(
map().addedCellMap(),
cellProcAddressing[addedI]
);
Info<< endl; renumber
(
map().addedFaceMap(),
faceProcAddressing[addedI]
);
renumber
(
map().addedPointMap(),
pointProcAddressing[addedI]
);
renumber
(
map().addedPatchMap(),
boundaryProcAddressing[addedI]
);
}
masterMesh.set(next, nullptr);
}
}
for (label proci=0; proci<nProcs; proci++)
{
Info<< "Reading mesh to add from "
<< databases[proci].caseName()
<< " for time = " << databases[proci].timeName()
<< nl << nl << endl;
} }
// See if any points on the mastermesh have become connected // See if any points on the mastermesh have become connected
// because of connections through processor meshes. // because of connections through processor meshes.
mergeSharedPoints(mergeDist, masterMesh, pointProcAddressing); mergeSharedPoints(mergeDist, masterMesh[0], pointProcAddressing);
// Save some properties on the reconstructed mesh // Save some properties on the reconstructed mesh
masterInternalFaces = masterMesh.nInternalFaces(); masterInternalFaces = masterMesh[0].nInternalFaces();
masterOwner = masterMesh.faceOwner(); masterOwner = masterMesh[0].faceOwner();
Info<< "\nWriting merged mesh to " Info<< "\nWriting merged mesh to "
<< runTime.path()/runTime.timeName() << runTime.path()/runTime.timeName()
<< nl << endl; << nl << endl;
if (!masterMesh.write()) if (!masterMesh[0].write())
{ {
FatalErrorInFunction FatalErrorInFunction
<< "Failed writing polyMesh." << "Failed writing polyMesh."
@ -731,7 +844,12 @@ int main(int argc, char *argv[])
if (writeCellDist) if (writeCellDist)
{ {
writeCellDistance(runTime, masterMesh, cellProcAddressing); writeCellDistance
(
runTime,
masterMesh[0],
cellProcAddressing
);
} }
} }
@ -808,7 +926,7 @@ int main(int argc, char *argv[])
// See reconstructPar for meaning of turning index. // See reconstructPar for meaning of turning index.
forAll(faceProcAddr, procFacei) forAll(faceProcAddr, procFacei)
{ {
label masterFacei = faceProcAddr[procFacei]; const label masterFacei = faceProcAddr[procFacei];
if if
( (