diff --git a/applications/utilities/mesh/generation/snappyHexMesh/Make/options b/applications/utilities/mesh/generation/snappyHexMesh/Make/options index 02177455ed..3c224d318f 100644 --- a/applications/utilities/mesh/generation/snappyHexMesh/Make/options +++ b/applications/utilities/mesh/generation/snappyHexMesh/Make/options @@ -11,7 +11,7 @@ EXE_INC = \ EXE_LIBS = \ -lfiniteVolume \ -ldecompositionMethods \ - -L$(FOAM_MPI_LIBBIN) -lptscotchDecomp -lparMetisDecomp -lmetisDecomp \ + -L$(FOAM_MPI_LIBBIN) -lptscotchDecomp \ -lmeshTools \ -ldynamicMesh \ -lautoMesh diff --git a/applications/utilities/mesh/generation/snappyHexMesh/snappyHexMesh.C b/applications/utilities/mesh/generation/snappyHexMesh/snappyHexMesh.C index 78f6947090..e9d637cddf 100644 --- a/applications/utilities/mesh/generation/snappyHexMesh/snappyHexMesh.C +++ b/applications/utilities/mesh/generation/snappyHexMesh/snappyHexMesh.C @@ -347,7 +347,7 @@ int main(int argc, char *argv[]) << "You have selected decomposition method " << decomposer.typeName << " which is not parallel aware." << endl - << "Please select one that is (hierarchical, ptscotch, parMetis)" + << "Please select one that is (hierarchical, ptscotch)" << exit(FatalError); } diff --git a/applications/utilities/parallelProcessing/redistributeMeshPar/Make/options b/applications/utilities/parallelProcessing/redistributeMeshPar/Make/options index f1839dafad..505234515d 100644 --- a/applications/utilities/parallelProcessing/redistributeMeshPar/Make/options +++ b/applications/utilities/parallelProcessing/redistributeMeshPar/Make/options @@ -7,6 +7,6 @@ EXE_INC = \ EXE_LIBS = \ -lfiniteVolume \ -ldecompositionMethods \ - -L$(FOAM_MPI_LIBBIN) -lptscotchDecomp -lparMetisDecomp -lmetisDecomp \ + -L$(FOAM_MPI_LIBBIN) -lptscotchDecomp \ -lmeshTools \ -ldynamicMesh diff --git a/src/dummyThirdParty/Allwmake b/src/dummyThirdParty/Allwmake index 20fc2517d2..c584d92194 100755 --- a/src/dummyThirdParty/Allwmake +++ b/src/dummyThirdParty/Allwmake @@ -5,7 +5,6 @@ set -x wmake libso scotchDecomp wmake libso ptscotchDecomp wmake libso metisDecomp -wmake libso parMetisDecomp wmake libso MGridGen diff --git a/src/dummyThirdParty/parMetisDecomp/Make/files b/src/dummyThirdParty/parMetisDecomp/Make/files deleted file mode 100644 index beb734541c..0000000000 --- a/src/dummyThirdParty/parMetisDecomp/Make/files +++ /dev/null @@ -1,3 +0,0 @@ -dummyParMetisDecomp.C - -LIB = $(FOAM_LIBBIN)/dummy/libparMetisDecomp diff --git a/src/dummyThirdParty/parMetisDecomp/Make/options b/src/dummyThirdParty/parMetisDecomp/Make/options deleted file mode 100644 index 2b87d62057..0000000000 --- a/src/dummyThirdParty/parMetisDecomp/Make/options +++ /dev/null @@ -1,5 +0,0 @@ -EXE_INC = \ - -I$(FOAM_SRC)/parallel/decompose/decompositionMethods/lnInclude \ - -I$(FOAM_SRC)/parallel/decompose/parMetisDecomp/lnInclude - -LIB_LIBS = diff --git a/src/dummyThirdParty/parMetisDecomp/dummyParMetisDecomp.C b/src/dummyThirdParty/parMetisDecomp/dummyParMetisDecomp.C deleted file mode 100644 index 9375c74398..0000000000 --- a/src/dummyThirdParty/parMetisDecomp/dummyParMetisDecomp.C +++ /dev/null @@ -1,166 +0,0 @@ -/*---------------------------------------------------------------------------*\ - ========= | - \\ / F ield | OpenFOAM: The Open Source CFD Toolbox - \\ / O peration | - \\ / A nd | Copyright (C) 2010-2010 OpenCFD Ltd. - \\/ M anipulation | -------------------------------------------------------------------------------- -License - This file is part of OpenFOAM. - - OpenFOAM is free software; you can redistribute it and/or modify it - under the terms of the GNU General Public License as published by the - Free Software Foundation; either version 2 of the License, or (at your - option) any later version. - - OpenFOAM is distributed in the hope that it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - for more details. - - You should have received a copy of the GNU General Public License - along with OpenFOAM; if not, write to the Free Software Foundation, - Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -\*---------------------------------------------------------------------------*/ - -#include "parMetisDecomp.H" -#include "addToRunTimeSelectionTable.H" -#include "polyMesh.H" -#include "Time.H" - -// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * // - -namespace Foam -{ - defineTypeNameAndDebug(parMetisDecomp, 0); - - addToRunTimeSelectionTable - ( - decompositionMethod, - parMetisDecomp, - dictionaryMesh - ); -} - -static const char* notImplementedMessage = -"You are trying to use parMetis but do not have the parMetisDecomp library " -"loaded.\n" -"This message is from the dummy parMetisDecomp stub library instead.\n" -"\n" -"Please install parMetis and make sure that libparMetis.so is in your " -"LD_LIBRARY_PATH.\n" -"The parMetisDecomp library can then be built in " -"$FOAM_SRC/parallel/decompose/decompositionMethods/parMetisDecomp\n"; - - -// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * // - -//- Does prevention of 0 cell domains and calls parmetis. -Foam::label Foam::parMetisDecomp::decompose -( - Field& xadj, - Field& adjncy, - const pointField& cellCentres, - Field& cellWeights, - Field& faceWeights, - const List& options, - - List& finalDecomp -) -{ - FatalErrorIn - ( - "label parMetisDecomp::decompose" - "(" - "Field&, " - "Field&, " - "const pointField&, " - "Field&, " - "Field&, " - "const List&, " - "List&" - ")" - )<< notImplementedMessage << exit(FatalError); - - return -1; -} - - -// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * // - -Foam::parMetisDecomp::parMetisDecomp -( - const dictionary& decompositionDict, - const polyMesh& mesh -) -: - decompositionMethod(decompositionDict), - mesh_(mesh) -{} - - -// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // - -Foam::labelList Foam::parMetisDecomp::decompose -( - const pointField& cc, - const scalarField& cWeights -) -{ - FatalErrorIn - ( - "labelList parMetisDecomp::decompose" - "(" - "const pointField&, " - "const scalarField&" - ")" - ) << notImplementedMessage << exit(FatalError); - - return labelList(); -} - - -Foam::labelList Foam::parMetisDecomp::decompose -( - const labelList& cellToRegion, - const pointField& regionPoints, - const scalarField& regionWeights -) -{ - FatalErrorIn - ( - "labelList parMetisDecomp::decompose" - "(" - "const labelList&, " - "const pointField&, " - "const scalarField&" - ")" - ) << notImplementedMessage << exit(FatalError); - - return labelList(); -} - - -Foam::labelList Foam::parMetisDecomp::decompose -( - const labelListList& globalCellCells, - const pointField& cellCentres, - const scalarField& cWeights -) -{ - FatalErrorIn - ( - "labelList parMetisDecomp::decompose" - "(" - "const labelListList&, " - "const pointField&, " - "const scalarField&" - ")" - ) << notImplementedMessage << exit(FatalError); - - return labelList(); -} - - -// ************************************************************************* // diff --git a/src/mesh/autoMesh/autoHexMesh/autoHexMeshDriver/autoHexMeshDriver.C b/src/mesh/autoMesh/autoHexMesh/autoHexMeshDriver/autoHexMeshDriver.C index 03284173da..93e52d6250 100644 --- a/src/mesh/autoMesh/autoHexMesh/autoHexMeshDriver/autoHexMeshDriver.C +++ b/src/mesh/autoMesh/autoHexMesh/autoHexMeshDriver/autoHexMeshDriver.C @@ -431,7 +431,7 @@ Foam::autoHexMeshDriver::autoHexMeshDriver << "You have selected decomposition method " << decomposer.typeName << " which is not parallel aware." << endl - << "Please select one that is (parMetis, hierarchical)" + << "Please select one that is (hierarchical, ptscotch)" << exit(FatalError); } diff --git a/src/parallel/decompose/Allwmake b/src/parallel/decompose/Allwmake index d39aecaa73..3ca612fb5b 100755 --- a/src/parallel/decompose/Allwmake +++ b/src/parallel/decompose/Allwmake @@ -9,7 +9,7 @@ wmake libso metisDecomp if [ -d "$FOAM_MPI_LIBBIN" ] then - ( WM_OPTIONS=${WM_OPTIONS}$WM_MPLIB; wmake libso ptscotchDecomp && wmake libso parMetisDecomp ) + ( WM_OPTIONS=${WM_OPTIONS}$WM_MPLIB; wmake libso ptscotchDecomp ) fi wmake libso decompositionMethods diff --git a/src/parallel/decompose/AllwmakeLnInclude b/src/parallel/decompose/AllwmakeLnInclude index 5a65c3a89c..c6a537aa42 100755 --- a/src/parallel/decompose/AllwmakeLnInclude +++ b/src/parallel/decompose/AllwmakeLnInclude @@ -4,7 +4,6 @@ set -x wmakeLnInclude decompositionMethods wmakeLnInclude metisDecomp -wmakeLnInclude parMetisDecomp wmakeLnInclude scotchDecomp wmakeLnInclude ptscotchDecomp diff --git a/src/parallel/decompose/decompositionMethods/Make/options b/src/parallel/decompose/decompositionMethods/Make/options index 197eb651c5..79be6f3a7d 100644 --- a/src/parallel/decompose/decompositionMethods/Make/options +++ b/src/parallel/decompose/decompositionMethods/Make/options @@ -1,9 +1,3 @@ EXE_INC = -LIB_LIBS = \ - /* -L$(FOAM_LIBBIN)/dummy */ \ - /* -L$(FOAM_MPI_LIBBIN) */ \ - /* -lscotchDecomp */ \ - /* -lmetisDecomp */ \ - /* -lparMetisDecomp */ - +LIB_LIBS = diff --git a/src/parallel/decompose/parMetisDecomp/Make/files b/src/parallel/decompose/parMetisDecomp/Make/files deleted file mode 100644 index 7b771b8161..0000000000 --- a/src/parallel/decompose/parMetisDecomp/Make/files +++ /dev/null @@ -1,3 +0,0 @@ -parMetisDecomp.C - -LIB = $(FOAM_MPI_LIBBIN)/libparMetisDecomp diff --git a/src/parallel/decompose/parMetisDecomp/Make/options b/src/parallel/decompose/parMetisDecomp/Make/options deleted file mode 100644 index 76a64be219..0000000000 --- a/src/parallel/decompose/parMetisDecomp/Make/options +++ /dev/null @@ -1,14 +0,0 @@ -include $(RULES)/mplib$(WM_MPLIB) - -EXE_INC = \ - $(PFLAGS) $(PINC) \ - -I$(WM_THIRD_PARTY_DIR)/ParMetis-3.1/ParMETISLib \ - -I$(WM_THIRD_PARTY_DIR)/ParMetis-3.1 \ - -I../decompositionMethods/lnInclude \ - -I../metisDecomp/lnInclude \ - -I../scotchDecomp/lnInclude - -LIB_LIBS = \ - -L$(FOAM_MPI_LIBBIN) \ - -lmetis-parmetis \ - -lparmetis diff --git a/src/parallel/decompose/parMetisDecomp/parMetisDecomp.C b/src/parallel/decompose/parMetisDecomp/parMetisDecomp.C deleted file mode 100644 index b339095d80..0000000000 --- a/src/parallel/decompose/parMetisDecomp/parMetisDecomp.C +++ /dev/null @@ -1,873 +0,0 @@ -/*---------------------------------------------------------------------------*\ - ========= | - \\ / F ield | OpenFOAM: The Open Source CFD Toolbox - \\ / O peration | - \\ / A nd | Copyright (C) 1991-2009 OpenCFD Ltd. - \\/ M anipulation | -------------------------------------------------------------------------------- -License - This file is part of OpenFOAM. - - OpenFOAM is free software; you can redistribute it and/or modify it - under the terms of the GNU General Public License as published by the - Free Software Foundation; either version 2 of the License, or (at your - option) any later version. - - OpenFOAM is distributed in the hope that it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - for more details. - - You should have received a copy of the GNU General Public License - along with OpenFOAM; if not, write to the Free Software Foundation, - Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -\*---------------------------------------------------------------------------*/ - -#include "parMetisDecomp.H" -#include "metisDecomp.H" -#include "syncTools.H" -#include "addToRunTimeSelectionTable.H" -#include "floatScalar.H" -#include "polyMesh.H" -#include "Time.H" -#include "labelIOField.H" -#include "globalIndex.H" - -#include - -extern "C" -{ -# include "parmetis.h" -} - -// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * // - -namespace Foam -{ - defineTypeNameAndDebug(parMetisDecomp, 0); - - addToRunTimeSelectionTable - ( - decompositionMethod, - parMetisDecomp, - dictionaryMesh - ); -} - - -// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * // - -//- Does prevention of 0 cell domains and calls parmetis. -Foam::label Foam::parMetisDecomp::decompose -( - Field& xadj, - Field& adjncy, - const pointField& cellCentres, - Field& cellWeights, - Field& faceWeights, - const List& options, - - List& finalDecomp -) -{ - // C style numbering - int numFlag = 0; - - // Number of dimensions - int nDims = 3; - - - if (cellCentres.size() != xadj.size()-1) - { - FatalErrorIn("parMetisDecomp::decompose(..)") - << "cellCentres:" << cellCentres.size() - << " xadj:" << xadj.size() - << abort(FatalError); - } - - - // Get number of cells on all processors - List nLocalCells(Pstream::nProcs()); - nLocalCells[Pstream::myProcNo()] = xadj.size()-1; - Pstream::gatherList(nLocalCells); - Pstream::scatterList(nLocalCells); - - // Get cell offsets. - List cellOffsets(Pstream::nProcs()+1); - int nGlobalCells = 0; - forAll(nLocalCells, procI) - { - cellOffsets[procI] = nGlobalCells; - nGlobalCells += nLocalCells[procI]; - } - cellOffsets[Pstream::nProcs()] = nGlobalCells; - - // Convert pointField into float - Field xyz(3*cellCentres.size()); - int compI = 0; - forAll(cellCentres, cellI) - { - const point& cc = cellCentres[cellI]; - xyz[compI++] = float(cc.x()); - xyz[compI++] = float(cc.y()); - xyz[compI++] = float(cc.z()); - } - - // Make sure every domain has at least one cell - // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - // (Metis falls over with zero sized domains) - // Trickle cells from processors that have them up to those that - // don't. - - - // Number of cells to send to the next processor - // (is same as number of cells next processor has to receive) - List nSendCells(Pstream::nProcs(), 0); - - for (label procI = nLocalCells.size()-1; procI >=1; procI--) - { - if (nLocalCells[procI]-nSendCells[procI] < 1) - { - nSendCells[procI-1] = nSendCells[procI]-nLocalCells[procI]+1; - } - } - - // First receive (so increasing the sizes of all arrays) - - if (Pstream::myProcNo() >= 1 && nSendCells[Pstream::myProcNo()-1] > 0) - { - // Receive cells from previous processor - IPstream fromPrevProc(Pstream::blocking, Pstream::myProcNo()-1); - - Field prevXadj(fromPrevProc); - Field prevAdjncy(fromPrevProc); - Field prevXyz(fromPrevProc); - Field prevCellWeights(fromPrevProc); - Field prevFaceWeights(fromPrevProc); - - if (prevXadj.size() != nSendCells[Pstream::myProcNo()-1]) - { - FatalErrorIn("parMetisDecomp::decompose(..)") - << "Expected from processor " << Pstream::myProcNo()-1 - << " connectivity for " << nSendCells[Pstream::myProcNo()-1] - << " nCells but only received " << prevXadj.size() - << abort(FatalError); - } - - // Insert adjncy - prepend(prevAdjncy, adjncy); - // Adapt offsets and prepend xadj - xadj += prevAdjncy.size(); - prepend(prevXadj, xadj); - // Coords - prepend(prevXyz, xyz); - // Weights - prepend(prevCellWeights, cellWeights); - prepend(prevFaceWeights, faceWeights); - } - - - // Send to my next processor - - if (nSendCells[Pstream::myProcNo()] > 0) - { - // Send cells to next processor - OPstream toNextProc(Pstream::blocking, Pstream::myProcNo()+1); - - int nCells = nSendCells[Pstream::myProcNo()]; - int startCell = xadj.size()-1 - nCells; - int startFace = xadj[startCell]; - int nFaces = adjncy.size()-startFace; - - // Send for all cell data: last nCells elements - // Send for all face data: last nFaces elements - toNextProc - << Field::subField(xadj, nCells, startCell)-startFace - << Field::subField(adjncy, nFaces, startFace) - << SubField(xyz, nDims*nCells, nDims*startCell) - << - ( - cellWeights.size() - ? static_cast&> - ( - Field::subField(cellWeights, nCells, startCell) - ) - : Field(0) - ) - << - ( - faceWeights.size() - ? static_cast&> - ( - Field::subField(faceWeights, nFaces, startFace) - ) - : Field(0) - ); - - // Remove data that has been sent - if (faceWeights.size()) - { - faceWeights.setSize(faceWeights.size()-nFaces); - } - if (cellWeights.size()) - { - cellWeights.setSize(cellWeights.size()-nCells); - } - xyz.setSize(xyz.size()-nDims*nCells); - adjncy.setSize(adjncy.size()-nFaces); - xadj.setSize(xadj.size() - nCells); - } - - - - // Adapt number of cells - forAll(nSendCells, procI) - { - // Sent cells - nLocalCells[procI] -= nSendCells[procI]; - - if (procI >= 1) - { - // Received cells - nLocalCells[procI] += nSendCells[procI-1]; - } - } - // Adapt cellOffsets - nGlobalCells = 0; - forAll(nLocalCells, procI) - { - cellOffsets[procI] = nGlobalCells; - nGlobalCells += nLocalCells[procI]; - } - - - if (nLocalCells[Pstream::myProcNo()] != (xadj.size()-1)) - { - FatalErrorIn("parMetisDecomp::decompose(..)") - << "Have connectivity for " << xadj.size()-1 - << " cells but nLocalCells:" << nLocalCells[Pstream::myProcNo()] - << abort(FatalError); - } - - // Weight info - int wgtFlag = 0; - int* vwgtPtr = NULL; - int* adjwgtPtr = NULL; - - if (cellWeights.size()) - { - vwgtPtr = cellWeights.begin(); - wgtFlag += 2; // Weights on vertices - } - if (faceWeights.size()) - { - adjwgtPtr = faceWeights.begin(); - wgtFlag += 1; // Weights on edges - } - - - // Number of weights or balance constraints - int nCon = 1; - // Per processor, per constraint the weight - Field tpwgts(nCon*nProcessors_, 1./nProcessors_); - // Imbalance tolerance - Field ubvec(nCon, 1.02); - if (nProcessors_ == 1) - { - // If only one processor there is no imbalance. - ubvec[0] = 1; - } - - MPI_Comm comm = MPI_COMM_WORLD; - - // output: cell -> processor addressing - finalDecomp.setSize(nLocalCells[Pstream::myProcNo()]); - - // output: number of cut edges - int edgeCut = 0; - - - ParMETIS_V3_PartGeomKway - ( - cellOffsets.begin(), // vtxDist - xadj.begin(), - adjncy.begin(), - vwgtPtr, // vertexweights - adjwgtPtr, // edgeweights - &wgtFlag, - &numFlag, - &nDims, - xyz.begin(), - &nCon, - &nProcessors_, // nParts - tpwgts.begin(), - ubvec.begin(), - const_cast&>(options).begin(), - &edgeCut, - finalDecomp.begin(), - &comm - ); - - - // If we sent cells across make sure we undo it - // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - // Receive back from next processor if I sent something - if (nSendCells[Pstream::myProcNo()] > 0) - { - IPstream fromNextProc(Pstream::blocking, Pstream::myProcNo()+1); - - List nextFinalDecomp(fromNextProc); - - if (nextFinalDecomp.size() != nSendCells[Pstream::myProcNo()]) - { - FatalErrorIn("parMetisDecomp::decompose(..)") - << "Expected from processor " << Pstream::myProcNo()+1 - << " decomposition for " << nSendCells[Pstream::myProcNo()] - << " nCells but only received " << nextFinalDecomp.size() - << abort(FatalError); - } - - append(nextFinalDecomp, finalDecomp); - } - - // Send back to previous processor. - if (Pstream::myProcNo() >= 1 && nSendCells[Pstream::myProcNo()-1] > 0) - { - OPstream toPrevProc(Pstream::blocking, Pstream::myProcNo()-1); - - int nToPrevious = nSendCells[Pstream::myProcNo()-1]; - - toPrevProc << - SubList - ( - finalDecomp, - nToPrevious, - finalDecomp.size()-nToPrevious - ); - - // Remove locally what has been sent - finalDecomp.setSize(finalDecomp.size()-nToPrevious); - } - - return edgeCut; -} - - -// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * // - -Foam::parMetisDecomp::parMetisDecomp -( - const dictionary& decompositionDict, - const polyMesh& mesh -) -: - decompositionMethod(decompositionDict), - mesh_(mesh) -{} - - -// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // - -Foam::labelList Foam::parMetisDecomp::decompose -( - const pointField& cc, - const scalarField& cWeights -) -{ - if (cc.size() != mesh_.nCells()) - { - FatalErrorIn - ( - "parMetisDecomp::decompose" - "(const pointField&, const scalarField&)" - ) << "Can use this decomposition method only for the whole mesh" - << endl - << "and supply one coordinate (cellCentre) for every cell." << endl - << "The number of coordinates " << cc.size() << endl - << "The number of cells in the mesh " << mesh_.nCells() - << exit(FatalError); - } - - // For running sequential ... - if (Pstream::nProcs() <= 1) - { - return metisDecomp(decompositionDict_, mesh_).decompose - ( - cc, - cWeights - ); - } - - - // Connections - Field adjncy; - // Offsets into adjncy - Field xadj; - calcDistributedCSR - ( - mesh_, - adjncy, - xadj - ); - - - // decomposition options. 0 = use defaults - List options(3, 0); - //options[0] = 1; // don't use defaults but use values below - //options[1] = -1; // full debug info - //options[2] = 15; // random number seed - - // cell weights (so on the vertices of the dual) - Field cellWeights; - - // face weights (so on the edges of the dual) - Field faceWeights; - - - // Check for externally provided cellweights and if so initialise weights - scalar minWeights = gMin(cWeights); - if (cWeights.size() > 0) - { - if (minWeights <= 0) - { - WarningIn - ( - "metisDecomp::decompose" - "(const pointField&, const scalarField&)" - ) << "Illegal minimum weight " << minWeights - << endl; - } - - if (cWeights.size() != mesh_.nCells()) - { - FatalErrorIn - ( - "parMetisDecomp::decompose" - "(const pointField&, const scalarField&)" - ) << "Number of cell weights " << cWeights.size() - << " does not equal number of cells " << mesh_.nCells() - << exit(FatalError); - } - - // Convert to integers. - cellWeights.setSize(cWeights.size()); - forAll(cellWeights, i) - { - cellWeights[i] = int(cWeights[i]/minWeights); - } - } - - - // Check for user supplied weights and decomp options - if (decompositionDict_.found("metisCoeffs")) - { - const dictionary& metisCoeffs = - decompositionDict_.subDict("metisCoeffs"); - word weightsFile; - - if (metisCoeffs.readIfPresent("cellWeightsFile", weightsFile)) - { - Info<< "parMetisDecomp : Using cell-based weights read from " - << weightsFile << endl; - - labelIOField cellIOWeights - ( - IOobject - ( - weightsFile, - mesh_.time().timeName(), - mesh_, - IOobject::MUST_READ, - IOobject::AUTO_WRITE - ) - ); - cellWeights.transfer(cellIOWeights); - - if (cellWeights.size() != mesh_.nCells()) - { - FatalErrorIn - ( - "parMetisDecomp::decompose" - "(const pointField&, const scalarField&)" - ) << "Number of cell weights " << cellWeights.size() - << " read from " << cellIOWeights.objectPath() - << " does not equal number of cells " << mesh_.nCells() - << exit(FatalError); - } - } - - if (metisCoeffs.readIfPresent("faceWeightsFile", weightsFile)) - { - Info<< "parMetisDecomp : Using face-based weights read from " - << weightsFile << endl; - - labelIOField weights - ( - IOobject - ( - weightsFile, - mesh_.time().timeName(), - mesh_, - IOobject::MUST_READ, - IOobject::AUTO_WRITE - ) - ); - - if (weights.size() != mesh_.nFaces()) - { - FatalErrorIn - ( - "parMetisDecomp::decompose" - "(const pointField&, const scalarField&)" - ) << "Number of face weights " << weights.size() - << " does not equal number of internal and boundary faces " - << mesh_.nFaces() - << exit(FatalError); - } - - faceWeights.setSize(adjncy.size()); - - // Assume symmetric weights. Keep same ordering as adjncy. - List nFacesPerCell(mesh_.nCells(), 0); - - // Handle internal faces - for (label faceI = 0; faceI < mesh_.nInternalFaces(); faceI++) - { - label w = weights[faceI]; - - label own = mesh_.faceOwner()[faceI]; - label nei = mesh_.faceNeighbour()[faceI]; - - faceWeights[xadj[own] + nFacesPerCell[own]++] = w; - faceWeights[xadj[nei] + nFacesPerCell[nei]++] = w; - } - // Coupled boundary faces - const polyBoundaryMesh& patches = mesh_.boundaryMesh(); - - forAll(patches, patchI) - { - const polyPatch& pp = patches[patchI]; - - if (pp.coupled()) - { - label faceI = pp.start(); - - forAll(pp, i) - { - label w = weights[faceI]; - label own = mesh_.faceOwner()[faceI]; - faceWeights[xadj[own] + nFacesPerCell[own]++] = w; - faceI++; - } - } - } - } - - if (metisCoeffs.readIfPresent("options", options)) - { - Info<< "Using Metis options " << options - << nl << endl; - - if (options.size() != 3) - { - FatalErrorIn - ( - "parMetisDecomp::decompose" - "(const pointField&, const scalarField&)" - ) << "Number of options " << options.size() - << " should be three." << exit(FatalError); - } - } - } - - - // Do actual decomposition - List finalDecomp; - decompose - ( - xadj, - adjncy, - cc, - cellWeights, - faceWeights, - options, - - finalDecomp - ); - - // Copy back to labelList - labelList decomp(finalDecomp.size()); - forAll(decomp, i) - { - decomp[i] = finalDecomp[i]; - } - return decomp; -} - - -Foam::labelList Foam::parMetisDecomp::decompose -( - const labelList& cellToRegion, - const pointField& regionPoints, - const scalarField& regionWeights -) -{ - const labelList& faceOwner = mesh_.faceOwner(); - const labelList& faceNeighbour = mesh_.faceNeighbour(); - const polyBoundaryMesh& patches = mesh_.boundaryMesh(); - - if (cellToRegion.size() != mesh_.nCells()) - { - FatalErrorIn - ( - "parMetisDecomp::decompose(const labelList&, const pointField&)" - ) << "Size of cell-to-coarse map " << cellToRegion.size() - << " differs from number of cells in mesh " << mesh_.nCells() - << exit(FatalError); - } - - - // Global region numbering engine - globalIndex globalRegions(regionPoints.size()); - - - // Get renumbered owner region on other side of coupled faces - // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - List globalNeighbour(mesh_.nFaces()-mesh_.nInternalFaces()); - - forAll(patches, patchI) - { - const polyPatch& pp = patches[patchI]; - - if (pp.coupled()) - { - label faceI = pp.start(); - label bFaceI = pp.start() - mesh_.nInternalFaces(); - - forAll(pp, i) - { - label ownRegion = cellToRegion[faceOwner[faceI]]; - globalNeighbour[bFaceI++] = globalRegions.toGlobal(ownRegion); - faceI++; - } - } - } - - // Get the cell on the other side of coupled patches - syncTools::swapBoundaryFaceList(mesh_, globalNeighbour, false); - - - // Get globalCellCells on coarse mesh - // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - labelListList globalRegionRegions; - { - List > dynRegionRegions(regionPoints.size()); - - // Internal faces first - forAll(faceNeighbour, faceI) - { - label ownRegion = cellToRegion[faceOwner[faceI]]; - label neiRegion = cellToRegion[faceNeighbour[faceI]]; - - if (ownRegion != neiRegion) - { - label globalOwn = globalRegions.toGlobal(ownRegion); - label globalNei = globalRegions.toGlobal(neiRegion); - - if (findIndex(dynRegionRegions[ownRegion], globalNei) == -1) - { - dynRegionRegions[ownRegion].append(globalNei); - } - if (findIndex(dynRegionRegions[neiRegion], globalOwn) == -1) - { - dynRegionRegions[neiRegion].append(globalOwn); - } - } - } - - // Coupled boundary faces - forAll(patches, patchI) - { - const polyPatch& pp = patches[patchI]; - - if (pp.coupled()) - { - label faceI = pp.start(); - label bFaceI = pp.start() - mesh_.nInternalFaces(); - - forAll(pp, i) - { - label ownRegion = cellToRegion[faceOwner[faceI]]; - label globalNei = globalNeighbour[bFaceI++]; - faceI++; - - if (findIndex(dynRegionRegions[ownRegion], globalNei) == -1) - { - dynRegionRegions[ownRegion].append(globalNei); - } - } - } - } - - globalRegionRegions.setSize(dynRegionRegions.size()); - forAll(dynRegionRegions, i) - { - globalRegionRegions[i].transfer(dynRegionRegions[i]); - } - } - - labelList regionDecomp - ( - decompose - ( - globalRegionRegions, - regionPoints, - regionWeights - ) - ); - - // Rework back into decomposition for original mesh_ - labelList cellDistribution(cellToRegion.size()); - - forAll(cellDistribution, cellI) - { - cellDistribution[cellI] = regionDecomp[cellToRegion[cellI]]; - } - return cellDistribution; -} - - -Foam::labelList Foam::parMetisDecomp::decompose -( - const labelListList& globalCellCells, - const pointField& cellCentres, - const scalarField& cWeights -) -{ - if (cellCentres.size() != globalCellCells.size()) - { - FatalErrorIn - ( - "parMetisDecomp::decompose(const labelListList&" - ", const pointField&, const scalarField&)" - ) << "Inconsistent number of cells (" << globalCellCells.size() - << ") and number of cell centres (" << cellCentres.size() - << ") or weights (" << cWeights.size() << ")." << exit(FatalError); - } - - // For running sequential ... - if (Pstream::nProcs() <= 1) - { - return metisDecomp(decompositionDict_, mesh_) - .decompose(globalCellCells, cellCentres, cWeights); - } - - - // Make Metis Distributed CSR (Compressed Storage Format) storage - - // Connections - Field adjncy; - // Offsets into adjncy - Field xadj; - calcCSR(globalCellCells, adjncy, xadj); - - // decomposition options. 0 = use defaults - List options(3, 0); - //options[0] = 1; // don't use defaults but use values below - //options[1] = -1; // full debug info - //options[2] = 15; // random number seed - - // cell weights (so on the vertices of the dual) - Field cellWeights; - - // face weights (so on the edges of the dual) - Field faceWeights; - - - // Check for externally provided cellweights and if so initialise weights - scalar minWeights = gMin(cWeights); - if (cWeights.size() > 0) - { - if (minWeights <= 0) - { - WarningIn - ( - "parMetisDecomp::decompose(const labelListList&" - ", const pointField&, const scalarField&)" - ) << "Illegal minimum weight " << minWeights - << endl; - } - - if (cWeights.size() != globalCellCells.size()) - { - FatalErrorIn - ( - "parMetisDecomp::decompose(const labelListList&" - ", const pointField&, const scalarField&)" - ) << "Number of cell weights " << cWeights.size() - << " does not equal number of cells " << globalCellCells.size() - << exit(FatalError); - } - - // Convert to integers. - cellWeights.setSize(cWeights.size()); - forAll(cellWeights, i) - { - cellWeights[i] = int(cWeights[i]/minWeights); - } - } - - - // Check for user supplied weights and decomp options - if (decompositionDict_.found("metisCoeffs")) - { - const dictionary& metisCoeffs = - decompositionDict_.subDict("metisCoeffs"); - - if (metisCoeffs.readIfPresent("options", options)) - { - Info<< "Using Metis options " << options - << nl << endl; - - if (options.size() != 3) - { - FatalErrorIn - ( - "parMetisDecomp::decompose(const labelListList&" - ", const pointField&, const scalarField&)" - ) << "Number of options " << options.size() - << " should be three." << exit(FatalError); - } - } - } - - - // Do actual decomposition - List finalDecomp; - decompose - ( - xadj, - adjncy, - cellCentres, - cellWeights, - faceWeights, - options, - - finalDecomp - ); - - // Copy back to labelList - labelList decomp(finalDecomp.size()); - forAll(decomp, i) - { - decomp[i] = finalDecomp[i]; - } - return decomp; -} - - -// ************************************************************************* // diff --git a/src/parallel/decompose/parMetisDecomp/parMetisDecomp.H b/src/parallel/decompose/parMetisDecomp/parMetisDecomp.H deleted file mode 100644 index 69c9500a47..0000000000 --- a/src/parallel/decompose/parMetisDecomp/parMetisDecomp.H +++ /dev/null @@ -1,175 +0,0 @@ -/*---------------------------------------------------------------------------*\ - ========= | - \\ / F ield | OpenFOAM: The Open Source CFD Toolbox - \\ / O peration | - \\ / A nd | Copyright (C) 1991-2009 OpenCFD Ltd. - \\/ M anipulation | -------------------------------------------------------------------------------- -License - This file is part of OpenFOAM. - - OpenFOAM is free software; you can redistribute it and/or modify it - under the terms of the GNU General Public License as published by the - Free Software Foundation; either version 2 of the License, or (at your - option) any later version. - - OpenFOAM is distributed in the hope that it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - for more details. - - You should have received a copy of the GNU General Public License - along with OpenFOAM; if not, write to the Free Software Foundation, - Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -Class - Foam::parMetisDecomp - -Description - -SourceFiles - parMetisDecomp.C - -\*---------------------------------------------------------------------------*/ - -#ifndef parMetisDecomp_H -#define parMetisDecomp_H - -#include "decompositionMethod.H" - -namespace Foam -{ - -/*---------------------------------------------------------------------------*\ - Class parMetisDecomp Declaration -\*---------------------------------------------------------------------------*/ - -class parMetisDecomp -: - public decompositionMethod -{ - // Private data - - const polyMesh& mesh_; - - - // Private Member Functions - - //- Insert list in front of list. - template - static void prepend(const UList&, List&); - //- Insert list at end of list. - template - static void append(const UList&, List&); - - label decompose - ( - Field& xadj, - Field& adjncy, - const pointField& cellCentres, - Field& cellWeights, - Field& faceWeights, - const List& options, - - List& finalDecomp - ); - - - //- Disallow default bitwise copy construct and assignment - void operator=(const parMetisDecomp&); - parMetisDecomp(const parMetisDecomp&); - - -public: - - //- Runtime type information - TypeName("parMetis"); - - - // Constructors - - //- Construct given the decomposition dictionary and mesh - parMetisDecomp - ( - const dictionary& decompositionDict, - const polyMesh& mesh - ); - - - // Destructor - - ~parMetisDecomp() - {} - - - // Member Functions - - //- parMetis handles Foam processor boundaries - virtual bool parallelAware() const - { - return true; - } - - //- Return for every coordinate the wanted processor number. Use the - // mesh connectivity (if needed) - // Weights get normalised so the minimum value is 1 before truncation - // to an integer so the weights should be multiples of the minimum - // value. The overall sum of weights might otherwise overflow. - virtual labelList decompose - ( - const pointField& points, - const scalarField& pointWeights - ); - - //- Return for every coordinate the wanted processor number. Gets - // passed agglomeration map (from fine to coarse cells) and coarse cell - // location. Can be overridden by decomposers that provide this - // functionality natively. - // See note on weights above. - virtual labelList decompose - ( - const labelList& cellToRegion, - const pointField& regionPoints, - const scalarField& regionWeights - ); - - //- Return for every coordinate the wanted processor number. Explicitly - // provided mesh connectivity. - // The connectivity is equal to mesh.cellCells() except for - // - in parallel the cell numbers are global cell numbers (starting - // from 0 at processor0 and then incrementing all through the - // processors) - // - the connections are across coupled patches - // See note on weights above. - virtual labelList decompose - ( - const labelListList& globalCellCells, - const pointField& cc, - const scalarField& cWeights - ); - - //- Helper to convert mesh connectivity into distributed CSR - static void calcMetisDistributedCSR - ( - const polyMesh&, - List& adjncy, - List& xadj - ); -}; - - -// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // - -} // End namespace Foam - -// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // - -#ifdef NoRepository -# include "parMetisDecompTemplates.C" -#endif - -// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // - -#endif - -// ************************************************************************* // diff --git a/src/parallel/decompose/parMetisDecomp/parMetisDecompTemplates.C b/src/parallel/decompose/parMetisDecomp/parMetisDecompTemplates.C deleted file mode 100644 index 8a883c8073..0000000000 --- a/src/parallel/decompose/parMetisDecomp/parMetisDecompTemplates.C +++ /dev/null @@ -1,76 +0,0 @@ -/*---------------------------------------------------------------------------*\ - ========= | - \\ / F ield | OpenFOAM: The Open Source CFD Toolbox - \\ / O peration | - \\ / A nd | Copyright (C) 1991-2009 OpenCFD Ltd. - \\/ M anipulation | -------------------------------------------------------------------------------- -License - This file is part of OpenFOAM. - - OpenFOAM is free software; you can redistribute it and/or modify it - under the terms of the GNU General Public License as published by the - Free Software Foundation; either version 2 of the License, or (at your - option) any later version. - - OpenFOAM is distributed in the hope that it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - for more details. - - You should have received a copy of the GNU General Public License - along with OpenFOAM; if not, write to the Free Software Foundation, - Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -\*---------------------------------------------------------------------------*/ - -#include "parMetisDecomp.H" - -// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // - -// Insert at front of list -template -void Foam::parMetisDecomp::prepend -( - const UList& extraLst, - List& lst -) -{ - label nExtra = extraLst.size(); - - // Make space for initial elements - lst.setSize(lst.size() + nExtra); - for (label i = lst.size()-1; i >= nExtra; i--) - { - lst[i] = lst[i-nExtra]; - } - - // Insert at front - forAll(extraLst, i) - { - lst[i] = extraLst[i]; - } -} - -// Insert at back of list -template -void Foam::parMetisDecomp::append -( - const UList& extraLst, - List& lst -) -{ - label sz = lst.size(); - - // Make space for initial elements - lst.setSize(sz + extraLst.size()); - - // Insert at back - forAll(extraLst, i) - { - lst[sz++] = extraLst[i]; - } -} - - -// ************************************************************************* // diff --git a/src/parallel/decompose/ptscotchDecomp/Make/options b/src/parallel/decompose/ptscotchDecomp/Make/options index 8c5ff0c5ad..f8e8f92460 100644 --- a/src/parallel/decompose/ptscotchDecomp/Make/options +++ b/src/parallel/decompose/ptscotchDecomp/Make/options @@ -4,9 +4,7 @@ EXE_INC = \ $(PFLAGS) $(PINC) \ -I$(WM_THIRD_PARTY_DIR)/scotch_5.1/include \ -I/usr/include/scotch \ - /* -I$(LIB_SRC)/parallel/decompose/parMetisDecomp */ \ - /* -I$(LIB_SRC)/parallel/decompose/scotchDecomp */ \ - -I$(LIB_SRC)/parallel/decompose/decompositionMethods/lnInclude + -I../decompositionMethods/lnInclude LIB_LIBS = \ -L$(FOAM_MPI_LIBBIN) -lptscotch -lptscotcherrexit diff --git a/src/parallel/decompose/scotchDecomp/scotchDecomp.C b/src/parallel/decompose/scotchDecomp/scotchDecomp.C index 3029fe3524..9f7e4b479a 100644 --- a/src/parallel/decompose/scotchDecomp/scotchDecomp.C +++ b/src/parallel/decompose/scotchDecomp/scotchDecomp.C @@ -474,7 +474,7 @@ Foam::labelList Foam::scotchDecomp::decompose { FatalErrorIn ( - "parMetisDecomp::decompose(const labelList&, const pointField&)" + "scotchDecomp::decompose(const labelList&, const pointField&)" ) << "Size of cell-to-coarse map " << agglom.size() << " differs from number of cells in mesh " << mesh_.nCells() << exit(FatalError); diff --git a/tutorials/incompressible/simpleFoam/motorBike/system/decomposeParDict b/tutorials/incompressible/simpleFoam/motorBike/system/decomposeParDict index bc80c7e6ad..c8c86b3fd8 100644 --- a/tutorials/incompressible/simpleFoam/motorBike/system/decomposeParDict +++ b/tutorials/incompressible/simpleFoam/motorBike/system/decomposeParDict @@ -19,7 +19,7 @@ numberOfSubdomains 6; method hierarchical; // method metis; -// method parMetis; +// method ptscotch; simpleCoeffs {