zhyang-dev/openfoam
1
0
Fork 0
mirror of https://develop.openfoam.com/Development/openfoam.git synced 2025-11-28 03:28:01 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

ENH: Removed parmetis. Use ptscotch instead.

Browse Source
This commit is contained in:
mattijs
2010-03-22 16:48:48 +00:00
parent 658b7700e1
commit 5d93fbe16d
19 changed files with 9 additions and 1334 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
applications/utilities/mesh/generation/snappyHexMesh/Make/options
View File

@ -11,7 +11,7 @@ EXE_INC = \
EXE_LIBS = \
-lfiniteVolume \
-ldecompositionMethods \
-L$(FOAM_MPI_LIBBIN) -lptscotchDecomp -lparMetisDecomp -lmetisDecomp \
-L$(FOAM_MPI_LIBBIN) -lptscotchDecomp \
-lmeshTools \
-ldynamicMesh \
-lautoMesh

2
applications/utilities/mesh/generation/snappyHexMesh/snappyHexMesh.C
View File

@ -347,7 +347,7 @@ int main(int argc, char *argv[])
<< "You have selected decomposition method "
<< decomposer.typeName
<< " which is not parallel aware." << endl
<< "Please select one that is (hierarchical, ptscotch, parMetis)"
<< "Please select one that is (hierarchical, ptscotch)"
<< exit(FatalError);
}

2
applications/utilities/parallelProcessing/redistributeMeshPar/Make/options
View File

@ -7,6 +7,6 @@ EXE_INC = \
EXE_LIBS = \
-lfiniteVolume \
-ldecompositionMethods \
-L$(FOAM_MPI_LIBBIN) -lptscotchDecomp -lparMetisDecomp -lmetisDecomp \
-L$(FOAM_MPI_LIBBIN) -lptscotchDecomp \
-lmeshTools \
-ldynamicMesh

1
src/dummyThirdParty/Allwmake vendored
View File

@ -5,7 +5,6 @@ set -x
wmake libso scotchDecomp
wmake libso ptscotchDecomp
wmake libso metisDecomp
wmake libso parMetisDecomp
wmake libso MGridGen

3
src/dummyThirdParty/parMetisDecomp/Make/files vendored
View File

@ -1,3 +0,0 @@
dummyParMetisDecomp.C
LIB = $(FOAM_LIBBIN)/dummy/libparMetisDecomp

5
src/dummyThirdParty/parMetisDecomp/Make/options vendored
View File

@ -1,5 +0,0 @@
EXE_INC = \
-I$(FOAM_SRC)/parallel/decompose/decompositionMethods/lnInclude \
-I$(FOAM_SRC)/parallel/decompose/parMetisDecomp/lnInclude
LIB_LIBS =

166
src/dummyThirdParty/parMetisDecomp/dummyParMetisDecomp.C vendored
View File

@ -1,166 +0,0 @@
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2010-2010 OpenCFD Ltd.
\\/ M anipulation |
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by the
Free Software Foundation; either version 2 of the License, or (at your
option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM; if not, write to the Free Software Foundation,
Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
\*---------------------------------------------------------------------------*/
#include "parMetisDecomp.H"
#include "addToRunTimeSelectionTable.H"
#include "polyMesh.H"
#include "Time.H"
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
namespace Foam
{
defineTypeNameAndDebug(parMetisDecomp, 0);
addToRunTimeSelectionTable
(
decompositionMethod,
parMetisDecomp,
dictionaryMesh
);
}
static const char* notImplementedMessage =
"You are trying to use parMetis but do not have the parMetisDecomp library "
"loaded.\n"
"This message is from the dummy parMetisDecomp stub library instead.\n"
"\n"
"Please install parMetis and make sure that libparMetis.so is in your "
"LD_LIBRARY_PATH.\n"
"The parMetisDecomp library can then be built in "
"$FOAM_SRC/parallel/decompose/decompositionMethods/parMetisDecomp\n";
// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
//- Does prevention of 0 cell domains and calls parmetis.
Foam::label Foam::parMetisDecomp::decompose
(
Field<int>& xadj,
Field<int>& adjncy,
const pointField& cellCentres,
Field<int>& cellWeights,
Field<int>& faceWeights,
const List<int>& options,
List<int>& finalDecomp
)
{
FatalErrorIn
(
"label parMetisDecomp::decompose"
"("
"Field<int>&, "
"Field<int>&, "
"const pointField&, "
"Field<int>&, "
"Field<int>&, "
"const List<int>&, "
"List<int>&"
")"
)<< notImplementedMessage << exit(FatalError);
return -1;
}
// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
Foam::parMetisDecomp::parMetisDecomp
(
const dictionary& decompositionDict,
const polyMesh& mesh
)
:
decompositionMethod(decompositionDict),
mesh_(mesh)
{}
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
Foam::labelList Foam::parMetisDecomp::decompose
(
const pointField& cc,
const scalarField& cWeights
)
{
FatalErrorIn
(
"labelList parMetisDecomp::decompose"
"("
"const pointField&, "
"const scalarField&"
")"
) << notImplementedMessage << exit(FatalError);
return labelList();
}
Foam::labelList Foam::parMetisDecomp::decompose
(
const labelList& cellToRegion,
const pointField& regionPoints,
const scalarField& regionWeights
)
{
FatalErrorIn
(
"labelList parMetisDecomp::decompose"
"("
"const labelList&, "
"const pointField&, "
"const scalarField&"
")"
) << notImplementedMessage << exit(FatalError);
return labelList();
}
Foam::labelList Foam::parMetisDecomp::decompose
(
const labelListList& globalCellCells,
const pointField& cellCentres,
const scalarField& cWeights
)
{
FatalErrorIn
(
"labelList parMetisDecomp::decompose"
"("
"const labelListList&, "
"const pointField&, "
"const scalarField&"
")"
) << notImplementedMessage << exit(FatalError);
return labelList();
}
// ************************************************************************* //

2
src/mesh/autoMesh/autoHexMesh/autoHexMeshDriver/autoHexMeshDriver.C
View File

@ -431,7 +431,7 @@ Foam::autoHexMeshDriver::autoHexMeshDriver
<< "You have selected decomposition method "
<< decomposer.typeName
<< " which is not parallel aware." << endl
<< "Please select one that is (parMetis, hierarchical)"
<< "Please select one that is (hierarchical, ptscotch)"
<< exit(FatalError);
}

2
src/parallel/decompose/Allwmake
View File

@ -9,7 +9,7 @@ wmake libso metisDecomp
if [ -d "$FOAM_MPI_LIBBIN" ]
then
( WM_OPTIONS=${WM_OPTIONS}$WM_MPLIB; wmake libso ptscotchDecomp && wmake libso parMetisDecomp )
( WM_OPTIONS=${WM_OPTIONS}$WM_MPLIB; wmake libso ptscotchDecomp )
fi
wmake libso decompositionMethods

1
src/parallel/decompose/AllwmakeLnInclude
View File

@ -4,7 +4,6 @@ set -x
wmakeLnInclude decompositionMethods
wmakeLnInclude metisDecomp
wmakeLnInclude parMetisDecomp
wmakeLnInclude scotchDecomp
wmakeLnInclude ptscotchDecomp

8
src/parallel/decompose/decompositionMethods/Make/options
View File

@ -1,9 +1,3 @@
EXE_INC =
LIB_LIBS = \
/* -L$(FOAM_LIBBIN)/dummy */ \
/* -L$(FOAM_MPI_LIBBIN) */ \
/* -lscotchDecomp */ \
/* -lmetisDecomp */ \
/* -lparMetisDecomp */
LIB_LIBS =

3
src/parallel/decompose/parMetisDecomp/Make/files
View File

@ -1,3 +0,0 @@
parMetisDecomp.C
LIB = $(FOAM_MPI_LIBBIN)/libparMetisDecomp

14
src/parallel/decompose/parMetisDecomp/Make/options
View File

@ -1,14 +0,0 @@
include $(RULES)/mplib$(WM_MPLIB)
EXE_INC = \
$(PFLAGS) $(PINC) \
-I$(WM_THIRD_PARTY_DIR)/ParMetis-3.1/ParMETISLib \
-I$(WM_THIRD_PARTY_DIR)/ParMetis-3.1 \
-I../decompositionMethods/lnInclude \
-I../metisDecomp/lnInclude \
-I../scotchDecomp/lnInclude
LIB_LIBS = \
-L$(FOAM_MPI_LIBBIN) \
-lmetis-parmetis \
-lparmetis

873
src/parallel/decompose/parMetisDecomp/parMetisDecomp.C
View File

@ -1,873 +0,0 @@
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 1991-2009 OpenCFD Ltd.
\\/ M anipulation |
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by the
Free Software Foundation; either version 2 of the License, or (at your
option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM; if not, write to the Free Software Foundation,
Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
\*---------------------------------------------------------------------------*/
#include "parMetisDecomp.H"
#include "metisDecomp.H"
#include "syncTools.H"
#include "addToRunTimeSelectionTable.H"
#include "floatScalar.H"
#include "polyMesh.H"
#include "Time.H"
#include "labelIOField.H"
#include "globalIndex.H"
#include <mpi.h>
extern "C"
{
# include "parmetis.h"
}
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
namespace Foam
{
defineTypeNameAndDebug(parMetisDecomp, 0);
addToRunTimeSelectionTable
(
decompositionMethod,
parMetisDecomp,
dictionaryMesh
);
}
// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
//- Does prevention of 0 cell domains and calls parmetis.
Foam::label Foam::parMetisDecomp::decompose
(
Field<int>& xadj,
Field<int>& adjncy,
const pointField& cellCentres,
Field<int>& cellWeights,
Field<int>& faceWeights,
const List<int>& options,
List<int>& finalDecomp
)
{
// C style numbering
int numFlag = 0;
// Number of dimensions
int nDims = 3;
if (cellCentres.size() != xadj.size()-1)
{
FatalErrorIn("parMetisDecomp::decompose(..)")
<< "cellCentres:" << cellCentres.size()
<< " xadj:" << xadj.size()
<< abort(FatalError);
}
// Get number of cells on all processors
List<int> nLocalCells(Pstream::nProcs());
nLocalCells[Pstream::myProcNo()] = xadj.size()-1;
Pstream::gatherList(nLocalCells);
Pstream::scatterList(nLocalCells);
// Get cell offsets.
List<int> cellOffsets(Pstream::nProcs()+1);
int nGlobalCells = 0;
forAll(nLocalCells, procI)
{
cellOffsets[procI] = nGlobalCells;
nGlobalCells += nLocalCells[procI];
}
cellOffsets[Pstream::nProcs()] = nGlobalCells;
// Convert pointField into float
Field<floatScalar> xyz(3*cellCentres.size());
int compI = 0;
forAll(cellCentres, cellI)
{
const point& cc = cellCentres[cellI];
xyz[compI++] = float(cc.x());
xyz[compI++] = float(cc.y());
xyz[compI++] = float(cc.z());
}
// Make sure every domain has at least one cell
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// (Metis falls over with zero sized domains)
// Trickle cells from processors that have them up to those that
// don't.
// Number of cells to send to the next processor
// (is same as number of cells next processor has to receive)
List<int> nSendCells(Pstream::nProcs(), 0);
for (label procI = nLocalCells.size()-1; procI >=1; procI--)
{
if (nLocalCells[procI]-nSendCells[procI] < 1)
{
nSendCells[procI-1] = nSendCells[procI]-nLocalCells[procI]+1;
}
}
// First receive (so increasing the sizes of all arrays)
if (Pstream::myProcNo() >= 1 && nSendCells[Pstream::myProcNo()-1] > 0)
{
// Receive cells from previous processor
IPstream fromPrevProc(Pstream::blocking, Pstream::myProcNo()-1);
Field<int> prevXadj(fromPrevProc);
Field<int> prevAdjncy(fromPrevProc);
Field<floatScalar> prevXyz(fromPrevProc);
Field<int> prevCellWeights(fromPrevProc);
Field<int> prevFaceWeights(fromPrevProc);
if (prevXadj.size() != nSendCells[Pstream::myProcNo()-1])
{
FatalErrorIn("parMetisDecomp::decompose(..)")
<< "Expected from processor " << Pstream::myProcNo()-1
<< " connectivity for " << nSendCells[Pstream::myProcNo()-1]
<< " nCells but only received " << prevXadj.size()
<< abort(FatalError);
}
// Insert adjncy
prepend(prevAdjncy, adjncy);
// Adapt offsets and prepend xadj
xadj += prevAdjncy.size();
prepend(prevXadj, xadj);
// Coords
prepend(prevXyz, xyz);
// Weights
prepend(prevCellWeights, cellWeights);
prepend(prevFaceWeights, faceWeights);
}
// Send to my next processor
if (nSendCells[Pstream::myProcNo()] > 0)
{
// Send cells to next processor
OPstream toNextProc(Pstream::blocking, Pstream::myProcNo()+1);
int nCells = nSendCells[Pstream::myProcNo()];
int startCell = xadj.size()-1 - nCells;
int startFace = xadj[startCell];
int nFaces = adjncy.size()-startFace;
// Send for all cell data: last nCells elements
// Send for all face data: last nFaces elements
toNextProc
<< Field<int>::subField(xadj, nCells, startCell)-startFace
<< Field<int>::subField(adjncy, nFaces, startFace)
<< SubField<floatScalar>(xyz, nDims*nCells, nDims*startCell)
<<
(
cellWeights.size()
? static_cast<const Field<int>&>
(
Field<int>::subField(cellWeights, nCells, startCell)
)
: Field<int>(0)
)
<<
(
faceWeights.size()
? static_cast<const Field<int>&>
(
Field<int>::subField(faceWeights, nFaces, startFace)
)
: Field<int>(0)
);
// Remove data that has been sent
if (faceWeights.size())
{
faceWeights.setSize(faceWeights.size()-nFaces);
}
if (cellWeights.size())
{
cellWeights.setSize(cellWeights.size()-nCells);
}
xyz.setSize(xyz.size()-nDims*nCells);
adjncy.setSize(adjncy.size()-nFaces);
xadj.setSize(xadj.size() - nCells);
}
// Adapt number of cells
forAll(nSendCells, procI)
{
// Sent cells
nLocalCells[procI] -= nSendCells[procI];
if (procI >= 1)
{
// Received cells
nLocalCells[procI] += nSendCells[procI-1];
}
}
// Adapt cellOffsets
nGlobalCells = 0;
forAll(nLocalCells, procI)
{
cellOffsets[procI] = nGlobalCells;
nGlobalCells += nLocalCells[procI];
}
if (nLocalCells[Pstream::myProcNo()] != (xadj.size()-1))
{
FatalErrorIn("parMetisDecomp::decompose(..)")
<< "Have connectivity for " << xadj.size()-1
<< " cells but nLocalCells:" << nLocalCells[Pstream::myProcNo()]
<< abort(FatalError);
}
// Weight info
int wgtFlag = 0;
int* vwgtPtr = NULL;
int* adjwgtPtr = NULL;
if (cellWeights.size())
{
vwgtPtr = cellWeights.begin();
wgtFlag += 2; // Weights on vertices
}
if (faceWeights.size())
{
adjwgtPtr = faceWeights.begin();
wgtFlag += 1; // Weights on edges
}
// Number of weights or balance constraints
int nCon = 1;
// Per processor, per constraint the weight
Field<floatScalar> tpwgts(nCon*nProcessors_, 1./nProcessors_);
// Imbalance tolerance
Field<floatScalar> ubvec(nCon, 1.02);
if (nProcessors_ == 1)
{
// If only one processor there is no imbalance.
ubvec[0] = 1;
}
MPI_Comm comm = MPI_COMM_WORLD;
// output: cell -> processor addressing
finalDecomp.setSize(nLocalCells[Pstream::myProcNo()]);
// output: number of cut edges
int edgeCut = 0;
ParMETIS_V3_PartGeomKway
(
cellOffsets.begin(), // vtxDist
xadj.begin(),
adjncy.begin(),
vwgtPtr, // vertexweights
adjwgtPtr, // edgeweights
&wgtFlag,
&numFlag,
&nDims,
xyz.begin(),
&nCon,
&nProcessors_, // nParts
tpwgts.begin(),
ubvec.begin(),
const_cast<List<int>&>(options).begin(),
&edgeCut,
finalDecomp.begin(),
&comm
);
// If we sent cells across make sure we undo it
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// Receive back from next processor if I sent something
if (nSendCells[Pstream::myProcNo()] > 0)
{
IPstream fromNextProc(Pstream::blocking, Pstream::myProcNo()+1);
List<int> nextFinalDecomp(fromNextProc);
if (nextFinalDecomp.size() != nSendCells[Pstream::myProcNo()])
{
FatalErrorIn("parMetisDecomp::decompose(..)")
<< "Expected from processor " << Pstream::myProcNo()+1
<< " decomposition for " << nSendCells[Pstream::myProcNo()]
<< " nCells but only received " << nextFinalDecomp.size()
<< abort(FatalError);
}
append(nextFinalDecomp, finalDecomp);
}
// Send back to previous processor.
if (Pstream::myProcNo() >= 1 && nSendCells[Pstream::myProcNo()-1] > 0)
{
OPstream toPrevProc(Pstream::blocking, Pstream::myProcNo()-1);
int nToPrevious = nSendCells[Pstream::myProcNo()-1];
toPrevProc <<
SubList<int>
(
finalDecomp,
nToPrevious,
finalDecomp.size()-nToPrevious
);
// Remove locally what has been sent
finalDecomp.setSize(finalDecomp.size()-nToPrevious);
}
return edgeCut;
}
// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
Foam::parMetisDecomp::parMetisDecomp
(
const dictionary& decompositionDict,
const polyMesh& mesh
)
:
decompositionMethod(decompositionDict),
mesh_(mesh)
{}
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
Foam::labelList Foam::parMetisDecomp::decompose
(
const pointField& cc,
const scalarField& cWeights
)
{
if (cc.size() != mesh_.nCells())
{
FatalErrorIn
(
"parMetisDecomp::decompose"
"(const pointField&, const scalarField&)"
) << "Can use this decomposition method only for the whole mesh"
<< endl
<< "and supply one coordinate (cellCentre) for every cell." << endl
<< "The number of coordinates " << cc.size() << endl
<< "The number of cells in the mesh " << mesh_.nCells()
<< exit(FatalError);
}
// For running sequential ...
if (Pstream::nProcs() <= 1)
{
return metisDecomp(decompositionDict_, mesh_).decompose
(
cc,
cWeights
);
}
// Connections
Field<int> adjncy;
// Offsets into adjncy
Field<int> xadj;
calcDistributedCSR
(
mesh_,
adjncy,
xadj
);
// decomposition options. 0 = use defaults
List<int> options(3, 0);
//options[0] = 1; // don't use defaults but use values below
//options[1] = -1; // full debug info
//options[2] = 15; // random number seed
// cell weights (so on the vertices of the dual)
Field<int> cellWeights;
// face weights (so on the edges of the dual)
Field<int> faceWeights;
// Check for externally provided cellweights and if so initialise weights
scalar minWeights = gMin(cWeights);
if (cWeights.size() > 0)
{
if (minWeights <= 0)
{
WarningIn
(
"metisDecomp::decompose"
"(const pointField&, const scalarField&)"
) << "Illegal minimum weight " << minWeights
<< endl;
}
if (cWeights.size() != mesh_.nCells())
{
FatalErrorIn
(
"parMetisDecomp::decompose"
"(const pointField&, const scalarField&)"
) << "Number of cell weights " << cWeights.size()
<< " does not equal number of cells " << mesh_.nCells()
<< exit(FatalError);
}
// Convert to integers.
cellWeights.setSize(cWeights.size());
forAll(cellWeights, i)
{
cellWeights[i] = int(cWeights[i]/minWeights);
}
}
// Check for user supplied weights and decomp options
if (decompositionDict_.found("metisCoeffs"))
{
const dictionary& metisCoeffs =
decompositionDict_.subDict("metisCoeffs");
word weightsFile;
if (metisCoeffs.readIfPresent("cellWeightsFile", weightsFile))
{
Info<< "parMetisDecomp : Using cell-based weights read from "
<< weightsFile << endl;
labelIOField cellIOWeights
(
IOobject
(
weightsFile,
mesh_.time().timeName(),
mesh_,
IOobject::MUST_READ,
IOobject::AUTO_WRITE
)
);
cellWeights.transfer(cellIOWeights);
if (cellWeights.size() != mesh_.nCells())
{
FatalErrorIn
(
"parMetisDecomp::decompose"
"(const pointField&, const scalarField&)"
) << "Number of cell weights " << cellWeights.size()
<< " read from " << cellIOWeights.objectPath()
<< " does not equal number of cells " << mesh_.nCells()
<< exit(FatalError);
}
}
if (metisCoeffs.readIfPresent("faceWeightsFile", weightsFile))
{
Info<< "parMetisDecomp : Using face-based weights read from "
<< weightsFile << endl;
labelIOField weights
(
IOobject
(
weightsFile,
mesh_.time().timeName(),
mesh_,
IOobject::MUST_READ,
IOobject::AUTO_WRITE
)
);
if (weights.size() != mesh_.nFaces())
{
FatalErrorIn
(
"parMetisDecomp::decompose"
"(const pointField&, const scalarField&)"
) << "Number of face weights " << weights.size()
<< " does not equal number of internal and boundary faces "
<< mesh_.nFaces()
<< exit(FatalError);
}
faceWeights.setSize(adjncy.size());
// Assume symmetric weights. Keep same ordering as adjncy.
List<int> nFacesPerCell(mesh_.nCells(), 0);
// Handle internal faces
for (label faceI = 0; faceI < mesh_.nInternalFaces(); faceI++)
{
label w = weights[faceI];
label own = mesh_.faceOwner()[faceI];
label nei = mesh_.faceNeighbour()[faceI];
faceWeights[xadj[own] + nFacesPerCell[own]++] = w;
faceWeights[xadj[nei] + nFacesPerCell[nei]++] = w;
}
// Coupled boundary faces
const polyBoundaryMesh& patches = mesh_.boundaryMesh();
forAll(patches, patchI)
{
const polyPatch& pp = patches[patchI];
if (pp.coupled())
{
label faceI = pp.start();
forAll(pp, i)
{
label w = weights[faceI];
label own = mesh_.faceOwner()[faceI];
faceWeights[xadj[own] + nFacesPerCell[own]++] = w;
faceI++;
}
}
}
}
if (metisCoeffs.readIfPresent("options", options))
{
Info<< "Using Metis options " << options
<< nl << endl;
if (options.size() != 3)
{
FatalErrorIn
(
"parMetisDecomp::decompose"
"(const pointField&, const scalarField&)"
) << "Number of options " << options.size()
<< " should be three." << exit(FatalError);
}
}
}
// Do actual decomposition
List<int> finalDecomp;
decompose
(
xadj,
adjncy,
cc,
cellWeights,
faceWeights,
options,
finalDecomp
);
// Copy back to labelList
labelList decomp(finalDecomp.size());
forAll(decomp, i)
{
decomp[i] = finalDecomp[i];
}
return decomp;
}
Foam::labelList Foam::parMetisDecomp::decompose
(
const labelList& cellToRegion,
const pointField& regionPoints,
const scalarField& regionWeights
)
{
const labelList& faceOwner = mesh_.faceOwner();
const labelList& faceNeighbour = mesh_.faceNeighbour();
const polyBoundaryMesh& patches = mesh_.boundaryMesh();
if (cellToRegion.size() != mesh_.nCells())
{
FatalErrorIn
(
"parMetisDecomp::decompose(const labelList&, const pointField&)"
) << "Size of cell-to-coarse map " << cellToRegion.size()
<< " differs from number of cells in mesh " << mesh_.nCells()
<< exit(FatalError);
}
// Global region numbering engine
globalIndex globalRegions(regionPoints.size());
// Get renumbered owner region on other side of coupled faces
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
List<int> globalNeighbour(mesh_.nFaces()-mesh_.nInternalFaces());
forAll(patches, patchI)
{
const polyPatch& pp = patches[patchI];
if (pp.coupled())
{
label faceI = pp.start();
label bFaceI = pp.start() - mesh_.nInternalFaces();
forAll(pp, i)
{
label ownRegion = cellToRegion[faceOwner[faceI]];
globalNeighbour[bFaceI++] = globalRegions.toGlobal(ownRegion);
faceI++;
}
}
}
// Get the cell on the other side of coupled patches
syncTools::swapBoundaryFaceList(mesh_, globalNeighbour, false);
// Get globalCellCells on coarse mesh
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
labelListList globalRegionRegions;
{
List<DynamicList<label> > dynRegionRegions(regionPoints.size());
// Internal faces first
forAll(faceNeighbour, faceI)
{
label ownRegion = cellToRegion[faceOwner[faceI]];
label neiRegion = cellToRegion[faceNeighbour[faceI]];
if (ownRegion != neiRegion)
{
label globalOwn = globalRegions.toGlobal(ownRegion);
label globalNei = globalRegions.toGlobal(neiRegion);
if (findIndex(dynRegionRegions[ownRegion], globalNei) == -1)
{
dynRegionRegions[ownRegion].append(globalNei);
}
if (findIndex(dynRegionRegions[neiRegion], globalOwn) == -1)
{
dynRegionRegions[neiRegion].append(globalOwn);
}
}
}
// Coupled boundary faces
forAll(patches, patchI)
{
const polyPatch& pp = patches[patchI];
if (pp.coupled())
{
label faceI = pp.start();
label bFaceI = pp.start() - mesh_.nInternalFaces();
forAll(pp, i)
{
label ownRegion = cellToRegion[faceOwner[faceI]];
label globalNei = globalNeighbour[bFaceI++];
faceI++;
if (findIndex(dynRegionRegions[ownRegion], globalNei) == -1)
{
dynRegionRegions[ownRegion].append(globalNei);
}
}
}
}
globalRegionRegions.setSize(dynRegionRegions.size());
forAll(dynRegionRegions, i)
{
globalRegionRegions[i].transfer(dynRegionRegions[i]);
}
}
labelList regionDecomp
(
decompose
(
globalRegionRegions,
regionPoints,
regionWeights
)
);
// Rework back into decomposition for original mesh_
labelList cellDistribution(cellToRegion.size());
forAll(cellDistribution, cellI)
{
cellDistribution[cellI] = regionDecomp[cellToRegion[cellI]];
}
return cellDistribution;
}
Foam::labelList Foam::parMetisDecomp::decompose
(
const labelListList& globalCellCells,
const pointField& cellCentres,
const scalarField& cWeights
)
{
if (cellCentres.size() != globalCellCells.size())
{
FatalErrorIn
(
"parMetisDecomp::decompose(const labelListList&"
", const pointField&, const scalarField&)"
) << "Inconsistent number of cells (" << globalCellCells.size()
<< ") and number of cell centres (" << cellCentres.size()
<< ") or weights (" << cWeights.size() << ")." << exit(FatalError);
}
// For running sequential ...
if (Pstream::nProcs() <= 1)
{
return metisDecomp(decompositionDict_, mesh_)
.decompose(globalCellCells, cellCentres, cWeights);
}
// Make Metis Distributed CSR (Compressed Storage Format) storage
// Connections
Field<int> adjncy;
// Offsets into adjncy
Field<int> xadj;
calcCSR(globalCellCells, adjncy, xadj);
// decomposition options. 0 = use defaults
List<int> options(3, 0);
//options[0] = 1; // don't use defaults but use values below
//options[1] = -1; // full debug info
//options[2] = 15; // random number seed
// cell weights (so on the vertices of the dual)
Field<int> cellWeights;
// face weights (so on the edges of the dual)
Field<int> faceWeights;
// Check for externally provided cellweights and if so initialise weights
scalar minWeights = gMin(cWeights);
if (cWeights.size() > 0)
{
if (minWeights <= 0)
{
WarningIn
(
"parMetisDecomp::decompose(const labelListList&"
", const pointField&, const scalarField&)"
) << "Illegal minimum weight " << minWeights
<< endl;
}
if (cWeights.size() != globalCellCells.size())
{
FatalErrorIn
(
"parMetisDecomp::decompose(const labelListList&"
", const pointField&, const scalarField&)"
) << "Number of cell weights " << cWeights.size()
<< " does not equal number of cells " << globalCellCells.size()
<< exit(FatalError);
}
// Convert to integers.
cellWeights.setSize(cWeights.size());
forAll(cellWeights, i)
{
cellWeights[i] = int(cWeights[i]/minWeights);
}
}
// Check for user supplied weights and decomp options
if (decompositionDict_.found("metisCoeffs"))
{
const dictionary& metisCoeffs =
decompositionDict_.subDict("metisCoeffs");
if (metisCoeffs.readIfPresent("options", options))
{
Info<< "Using Metis options " << options
<< nl << endl;
if (options.size() != 3)
{
FatalErrorIn
(
"parMetisDecomp::decompose(const labelListList&"
", const pointField&, const scalarField&)"
) << "Number of options " << options.size()
<< " should be three." << exit(FatalError);
}
}
}
// Do actual decomposition
List<int> finalDecomp;
decompose
(
xadj,
adjncy,
cellCentres,
cellWeights,
faceWeights,
options,
finalDecomp
);
// Copy back to labelList
labelList decomp(finalDecomp.size());
forAll(decomp, i)
{
decomp[i] = finalDecomp[i];
}
return decomp;
}
// ************************************************************************* //

175
src/parallel/decompose/parMetisDecomp/parMetisDecomp.H
View File

@ -1,175 +0,0 @@
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 1991-2009 OpenCFD Ltd.
\\/ M anipulation |
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by the
Free Software Foundation; either version 2 of the License, or (at your
option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM; if not, write to the Free Software Foundation,
Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
Class
Foam::parMetisDecomp
Description
SourceFiles
parMetisDecomp.C
\*---------------------------------------------------------------------------*/
#ifndef parMetisDecomp_H
#define parMetisDecomp_H
#include "decompositionMethod.H"
namespace Foam
{
/*---------------------------------------------------------------------------*\
Class parMetisDecomp Declaration
\*---------------------------------------------------------------------------*/
class parMetisDecomp
:
public decompositionMethod
{
// Private data
const polyMesh& mesh_;
// Private Member Functions
//- Insert list in front of list.
template<class Type>
static void prepend(const UList<Type>&, List<Type>&);
//- Insert list at end of list.
template<class Type>
static void append(const UList<Type>&, List<Type>&);
label decompose
(
Field<int>& xadj,
Field<int>& adjncy,
const pointField& cellCentres,
Field<int>& cellWeights,
Field<int>& faceWeights,
const List<int>& options,
List<int>& finalDecomp
);
//- Disallow default bitwise copy construct and assignment
void operator=(const parMetisDecomp&);
parMetisDecomp(const parMetisDecomp&);
public:
//- Runtime type information
TypeName("parMetis");
// Constructors
//- Construct given the decomposition dictionary and mesh
parMetisDecomp
(
const dictionary& decompositionDict,
const polyMesh& mesh
);
// Destructor
~parMetisDecomp()
{}
// Member Functions
//- parMetis handles Foam processor boundaries
virtual bool parallelAware() const
{
return true;
}
//- Return for every coordinate the wanted processor number. Use the
// mesh connectivity (if needed)
// Weights get normalised so the minimum value is 1 before truncation
// to an integer so the weights should be multiples of the minimum
// value. The overall sum of weights might otherwise overflow.
virtual labelList decompose
(
const pointField& points,
const scalarField& pointWeights
);
//- Return for every coordinate the wanted processor number. Gets
// passed agglomeration map (from fine to coarse cells) and coarse cell
// location. Can be overridden by decomposers that provide this
// functionality natively.
// See note on weights above.
virtual labelList decompose
(
const labelList& cellToRegion,
const pointField& regionPoints,
const scalarField& regionWeights
);
//- Return for every coordinate the wanted processor number. Explicitly
// provided mesh connectivity.
// The connectivity is equal to mesh.cellCells() except for
// - in parallel the cell numbers are global cell numbers (starting
// from 0 at processor0 and then incrementing all through the
// processors)
// - the connections are across coupled patches
// See note on weights above.
virtual labelList decompose
(
const labelListList& globalCellCells,
const pointField& cc,
const scalarField& cWeights
);
//- Helper to convert mesh connectivity into distributed CSR
static void calcMetisDistributedCSR
(
const polyMesh&,
List<int>& adjncy,
List<int>& xadj
);
};
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
} // End namespace Foam
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
#ifdef NoRepository
# include "parMetisDecompTemplates.C"
#endif
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
#endif
// ************************************************************************* //

76
src/parallel/decompose/parMetisDecomp/parMetisDecompTemplates.C
View File

@ -1,76 +0,0 @@
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 1991-2009 OpenCFD Ltd.
\\/ M anipulation |
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by the
Free Software Foundation; either version 2 of the License, or (at your
option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM; if not, write to the Free Software Foundation,
Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
\*---------------------------------------------------------------------------*/
#include "parMetisDecomp.H"
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
// Insert at front of list
template<class Type>
void Foam::parMetisDecomp::prepend
(
const UList<Type>& extraLst,
List<Type>& lst
)
{
label nExtra = extraLst.size();
// Make space for initial elements
lst.setSize(lst.size() + nExtra);
for (label i = lst.size()-1; i >= nExtra; i--)
{
lst[i] = lst[i-nExtra];
}
// Insert at front
forAll(extraLst, i)
{
lst[i] = extraLst[i];
}
}
// Insert at back of list
template<class Type>
void Foam::parMetisDecomp::append
(
const UList<Type>& extraLst,
List<Type>& lst
)
{
label sz = lst.size();
// Make space for initial elements
lst.setSize(sz + extraLst.size());
// Insert at back
forAll(extraLst, i)
{
lst[sz++] = extraLst[i];
}
}
// ************************************************************************* //

4
src/parallel/decompose/ptscotchDecomp/Make/options
View File

@ -4,9 +4,7 @@ EXE_INC = \
$(PFLAGS) $(PINC) \
-I$(WM_THIRD_PARTY_DIR)/scotch_5.1/include \
-I/usr/include/scotch \
/* -I$(LIB_SRC)/parallel/decompose/parMetisDecomp */ \
/* -I$(LIB_SRC)/parallel/decompose/scotchDecomp */ \
-I$(LIB_SRC)/parallel/decompose/decompositionMethods/lnInclude
-I../decompositionMethods/lnInclude
LIB_LIBS = \
-L$(FOAM_MPI_LIBBIN) -lptscotch -lptscotcherrexit

2
src/parallel/decompose/scotchDecomp/scotchDecomp.C
View File

@ -474,7 +474,7 @@ Foam::labelList Foam::scotchDecomp::decompose
{
FatalErrorIn
(
"parMetisDecomp::decompose(const labelList&, const pointField&)"
"scotchDecomp::decompose(const labelList&, const pointField&)"
) << "Size of cell-to-coarse map " << agglom.size()
<< " differs from number of cells in mesh " << mesh_.nCells()
<< exit(FatalError);

2
tutorials/incompressible/simpleFoam/motorBike/system/decomposeParDict
View File

@ -19,7 +19,7 @@ numberOfSubdomains 6;
method hierarchical;
// method metis;
// method parMetis;
// method ptscotch;
simpleCoeffs
{