diff --git a/src/mesh/autoMesh/Make/options b/src/mesh/autoMesh/Make/options index 573d0007c1..28c3d5fdf0 100644 --- a/src/mesh/autoMesh/Make/options +++ b/src/mesh/autoMesh/Make/options @@ -8,7 +8,6 @@ EXE_INC = \ -I$(LIB_SRC)/triSurface/lnInclude LIB_LIBS = \ - -ldecompositionMethods \ -ldynamicMesh \ -lfiniteVolume \ -llagrangian \ diff --git a/src/meshTools/Make/options b/src/meshTools/Make/options index 0ff2202926..6bf989d4f5 100644 --- a/src/meshTools/Make/options +++ b/src/meshTools/Make/options @@ -6,4 +6,5 @@ EXE_INC = \ LIB_LIBS = \ -ltriSurface \ -ldecompositionMethods \ + -lmetisDecompositionMethod \ -llagrangian diff --git a/src/parallel/Allwmake b/src/parallel/Allwmake index 34d323251a..48c7d41624 100755 --- a/src/parallel/Allwmake +++ b/src/parallel/Allwmake @@ -3,6 +3,7 @@ cd ${0%/*} || exit 1 # run from this directory set -x wmake libso decompositionMethods +wmake libso metisDecomp wmake libso reconstruct if [ -d "$FOAM_MPI_LIBBIN" ] diff --git a/src/parallel/decompositionMethods/Make/files b/src/parallel/decompositionMethods/Make/files index 2acbfd5606..751ebe90f2 100644 --- a/src/parallel/decompositionMethods/Make/files +++ b/src/parallel/decompositionMethods/Make/files @@ -6,6 +6,4 @@ manualDecomp/manualDecomp.C scotchDecomp/scotchDecomp.C -metisDecomp/metisDecomp.C - LIB = $(FOAM_LIBBIN)/libdecompositionMethods diff --git a/src/parallel/decompositionMethods/Make/options b/src/parallel/decompositionMethods/Make/options index 97569aa663..fe6e8b6a05 100644 --- a/src/parallel/decompositionMethods/Make/options +++ b/src/parallel/decompositionMethods/Make/options @@ -1,8 +1,6 @@ EXE_INC = \ - -I$(WM_THIRD_PARTY_DIR)/scotch_5.1/src/libscotch/lnInclude \ - -I$(WM_THIRD_PARTY_DIR)/metis-5.0pre2/include + -I$(WM_THIRD_PARTY_DIR)/scotch_5.1/include \ + -I../metisDecomp/lnInclude LIB_LIBS = \ - -lscotch \ - -lmetis \ - -lGKlib + -lscotch -lscotcherrexit diff --git a/src/parallel/decompositionMethods/scotchDecomp/scotchDecomp.C b/src/parallel/decompositionMethods/scotchDecomp/scotchDecomp.C index 61a9483837..5375fb529b 100644 --- a/src/parallel/decompositionMethods/scotchDecomp/scotchDecomp.C +++ b/src/parallel/decompositionMethods/scotchDecomp/scotchDecomp.C @@ -61,17 +61,12 @@ License #include "scotchDecomp.H" #include "addToRunTimeSelectionTable.H" #include "floatScalar.H" -#include "IFstream.H" #include "Time.H" #include "cyclicPolyPatch.H" #include "OFstream.H" -#include "metisDecomp.H" extern "C" { -#define OMPI_SKIP_MPICXX -#include "module.h" -#include "common.h" #include "scotch.h" } @@ -115,13 +110,13 @@ void Foam::scotchDecomp::check(const int retVal, const char* str) if (retVal) { FatalErrorIn("scotchDecomp::decompose(..)") - << "Called to scotch routine " << str << " failed." + << "Call to scotch routine " << str << " failed." << exit(FatalError); } } -// Call Metis with options from dictionary. +// Call scotch with options from dictionary. Foam::label Foam::scotchDecomp::decompose ( const List& adjncy, @@ -173,7 +168,7 @@ Foam::label Foam::scotchDecomp::decompose { FatalErrorIn ( - "parMetisDecomp::decompose" + "scotchDecomp::decompose" "(const pointField&, const scalarField&)" ) << "Number of cell weights " << cWeights.size() << " does not equal number of cells " << xadj.size()-1 @@ -377,7 +372,10 @@ Foam::labelList Foam::scotchDecomp::decompose { if (points.size() != mesh_.nCells()) { - FatalErrorIn("scotchDecomp::decompose(const pointField&)") + FatalErrorIn + ( + "scotchDecomp::decompose(const pointField&, const scalarField&)" + ) << "Can use this decomposition method only for the whole mesh" << endl << "and supply one coordinate (cellCentre) for every cell." << endl @@ -391,12 +389,7 @@ Foam::labelList Foam::scotchDecomp::decompose // xadj(celli) : start of information in adjncy for celli List adjncy; List xadj; - metisDecomp::calcMetisCSR - ( - mesh_, - adjncy, - xadj - ); + calcCSR(mesh_, adjncy, xadj); // Decompose using default weights List finalDecomp; @@ -445,7 +438,7 @@ Foam::labelList Foam::scotchDecomp::decompose cellCells ); - metisDecomp::calcMetisCSR(cellCells, adjncy, xadj); + calcCSR(cellCells, adjncy, xadj); } // Decompose using weights @@ -475,7 +468,8 @@ Foam::labelList Foam::scotchDecomp::decompose { FatalErrorIn ( - "scotchDecomp::decompose(const pointField&, const labelListList&)" + "scotchDecomp::decompose" + "(const labelListList&, const pointField&, const scalarField&)" ) << "Inconsistent number of cells (" << globalCellCells.size() << ") and number of cell centres (" << cellCentres.size() << ")." << exit(FatalError); @@ -488,7 +482,7 @@ Foam::labelList Foam::scotchDecomp::decompose List adjncy; List xadj; - metisDecomp::calcMetisCSR(globalCellCells, adjncy, xadj); + calcCSR(globalCellCells, adjncy, xadj); // Decompose using weights List finalDecomp; @@ -504,4 +498,144 @@ Foam::labelList Foam::scotchDecomp::decompose } +void Foam::scotchDecomp::calcCSR +( + const polyMesh& mesh, + List& adjncy, + List& xadj +) +{ + // Make Metis CSR (Compressed Storage Format) storage + // adjncy : contains neighbours (= edges in graph) + // xadj(celli) : start of information in adjncy for celli + + xadj.setSize(mesh.nCells()+1); + + // Initialise the number of internal faces of the cells to twice the + // number of internal faces + label nInternalFaces = 2*mesh.nInternalFaces(); + + // Check the boundary for coupled patches and add to the number of + // internal faces + const polyBoundaryMesh& pbm = mesh.boundaryMesh(); + + forAll(pbm, patchi) + { + if (isA(pbm[patchi])) + { + nInternalFaces += pbm[patchi].size(); + } + } + + // Create the adjncy array the size of the total number of internal and + // coupled faces + adjncy.setSize(nInternalFaces); + + // Fill in xadj + // ~~~~~~~~~~~~ + label freeAdj = 0; + + for (label cellI = 0; cellI < mesh.nCells(); cellI++) + { + xadj[cellI] = freeAdj; + + const labelList& cFaces = mesh.cells()[cellI]; + + forAll(cFaces, i) + { + label faceI = cFaces[i]; + + if + ( + mesh.isInternalFace(faceI) + || isA(pbm[pbm.whichPatch(faceI)]) + ) + { + freeAdj++; + } + } + } + xadj[mesh.nCells()] = freeAdj; + + + // Fill in adjncy + // ~~~~~~~~~~~~~~ + + labelList nFacesPerCell(mesh.nCells(), 0); + + // Internal faces + for (label faceI = 0; faceI < mesh.nInternalFaces(); faceI++) + { + label own = mesh.faceOwner()[faceI]; + label nei = mesh.faceNeighbour()[faceI]; + + adjncy[xadj[own] + nFacesPerCell[own]++] = nei; + adjncy[xadj[nei] + nFacesPerCell[nei]++] = own; + } + + // Coupled faces. Only cyclics done. + forAll(pbm, patchi) + { + if (isA(pbm[patchi])) + { + const unallocLabelList& faceCells = pbm[patchi].faceCells(); + + label sizeby2 = faceCells.size()/2; + + for (label facei=0; facei& adjncy, + List& xadj +) +{ + // Count number of internal faces + label nConnections = 0; + + forAll(cellCells, coarseI) + { + nConnections += cellCells[coarseI].size(); + } + + // Create the adjncy array as twice the size of the total number of + // internal faces + adjncy.setSize(nConnections); + + xadj.setSize(cellCells.size()+1); + + + // Fill in xadj + // ~~~~~~~~~~~~ + label freeAdj = 0; + + forAll(cellCells, coarseI) + { + xadj[coarseI] = freeAdj; + + const labelList& cCells = cellCells[coarseI]; + + forAll(cCells, i) + { + adjncy[freeAdj++] = cCells[i]; + } + } + xadj[cellCells.size()] = freeAdj; +} + + + // ************************************************************************* // diff --git a/src/parallel/decompositionMethods/scotchDecomp/scotchDecomp.H b/src/parallel/decompositionMethods/scotchDecomp/scotchDecomp.H index 54efb1bafa..93bcc7b9a4 100644 --- a/src/parallel/decompositionMethods/scotchDecomp/scotchDecomp.H +++ b/src/parallel/decompositionMethods/scotchDecomp/scotchDecomp.H @@ -135,6 +135,25 @@ public: const scalarField& cWeights ); + + //- Helper to convert local connectivity (supplied as owner,neighbour) + // into CSR (Metis,scotch) storage. Does cyclics but not processor + // patches + static void calcCSR + ( + const polyMesh& mesh, + List& adjncy, + List& xadj + ); + + //- Helper to convert connectivity (supplied as cellcells) into + // CSR (Metis,scotch) storage + static void calcCSR + ( + const labelListList& globalCellCells, + List& adjncy, + List& xadj + ); }; diff --git a/src/parallel/metisDecomp/Make/files b/src/parallel/metisDecomp/Make/files new file mode 100644 index 0000000000..eb1e633e18 --- /dev/null +++ b/src/parallel/metisDecomp/Make/files @@ -0,0 +1,3 @@ +metisDecomp.C + +LIB = $(FOAM_LIBBIN)/libmetisDecompositionMethod diff --git a/src/parallel/metisDecomp/Make/options b/src/parallel/metisDecomp/Make/options new file mode 100644 index 0000000000..a7d5398f03 --- /dev/null +++ b/src/parallel/metisDecomp/Make/options @@ -0,0 +1,7 @@ +EXE_INC = \ + -I$(WM_THIRD_PARTY_DIR)/metis-5.0pre2/include \ + -I../decompositionMethods/lnInclude + +LIB_LIBS = \ + -lmetis \ + -lGKlib diff --git a/src/parallel/decompositionMethods/metisDecomp/metisDecomp.C b/src/parallel/metisDecomp/metisDecomp.C similarity index 78% rename from src/parallel/decompositionMethods/metisDecomp/metisDecomp.C rename to src/parallel/metisDecomp/metisDecomp.C index 3b2c5fb58c..9066e92248 100644 --- a/src/parallel/decompositionMethods/metisDecomp/metisDecomp.C +++ b/src/parallel/metisDecomp/metisDecomp.C @@ -27,9 +27,8 @@ License #include "metisDecomp.H" #include "addToRunTimeSelectionTable.H" #include "floatScalar.H" -#include "IFstream.H" #include "Time.H" -#include "cyclicPolyPatch.H" +#include "scotchDecomp.H" extern "C" { @@ -331,12 +330,7 @@ Foam::labelList Foam::metisDecomp::decompose List adjncy; List xadj; - calcMetisCSR - ( - mesh_, - adjncy, - xadj - ); + scotchDecomp::calcCSR(mesh_, adjncy, xadj); // Decompose using default weights List finalDecomp; @@ -352,145 +346,6 @@ Foam::labelList Foam::metisDecomp::decompose } -void Foam::metisDecomp::calcMetisCSR -( - const polyMesh& mesh, - List& adjncy, - List& xadj -) -{ - // Make Metis CSR (Compressed Storage Format) storage - // adjncy : contains neighbours (= edges in graph) - // xadj(celli) : start of information in adjncy for celli - - xadj.setSize(mesh.nCells()+1); - - // Initialise the number of internal faces of the cells to twice the - // number of internal faces - label nInternalFaces = 2*mesh.nInternalFaces(); - - // Check the boundary for coupled patches and add to the number of - // internal faces - const polyBoundaryMesh& pbm = mesh.boundaryMesh(); - - forAll(pbm, patchi) - { - if (isA(pbm[patchi])) - { - nInternalFaces += pbm[patchi].size(); - } - } - - // Create the adjncy array the size of the total number of internal and - // coupled faces - adjncy.setSize(nInternalFaces); - - // Fill in xadj - // ~~~~~~~~~~~~ - label freeAdj = 0; - - for (label cellI = 0; cellI < mesh.nCells(); cellI++) - { - xadj[cellI] = freeAdj; - - const labelList& cFaces = mesh.cells()[cellI]; - - forAll(cFaces, i) - { - label faceI = cFaces[i]; - - if - ( - mesh.isInternalFace(faceI) - || isA(pbm[pbm.whichPatch(faceI)]) - ) - { - freeAdj++; - } - } - } - xadj[mesh.nCells()] = freeAdj; - - - // Fill in adjncy - // ~~~~~~~~~~~~~~ - - labelList nFacesPerCell(mesh.nCells(), 0); - - // Internal faces - for (label faceI = 0; faceI < mesh.nInternalFaces(); faceI++) - { - label own = mesh.faceOwner()[faceI]; - label nei = mesh.faceNeighbour()[faceI]; - - adjncy[xadj[own] + nFacesPerCell[own]++] = nei; - adjncy[xadj[nei] + nFacesPerCell[nei]++] = own; - } - - // Coupled faces. Only cyclics done. - forAll(pbm, patchi) - { - if (isA(pbm[patchi])) - { - const unallocLabelList& faceCells = pbm[patchi].faceCells(); - - label sizeby2 = faceCells.size()/2; - - for (label facei=0; facei& adjncy, - List& xadj -) -{ - // Count number of internal faces - label nConnections = 0; - - forAll(cellCells, coarseI) - { - nConnections += cellCells[coarseI].size(); - } - - // Create the adjncy array as twice the size of the total number of - // internal faces - adjncy.setSize(nConnections); - - xadj.setSize(cellCells.size()+1); - - - // Fill in xadj - // ~~~~~~~~~~~~ - label freeAdj = 0; - - forAll(cellCells, coarseI) - { - xadj[coarseI] = freeAdj; - - const labelList& cCells = cellCells[coarseI]; - - forAll(cCells, i) - { - adjncy[freeAdj++] = cCells[i]; - } - } - xadj[cellCells.size()] = freeAdj; -} - - Foam::labelList Foam::metisDecomp::decompose ( const labelList& agglom, @@ -525,7 +380,7 @@ Foam::labelList Foam::metisDecomp::decompose cellCells ); - calcMetisCSR(cellCells, adjncy, xadj); + scotchDecomp::calcCSR(cellCells, adjncy, xadj); } // Decompose using default weights @@ -570,7 +425,7 @@ Foam::labelList Foam::metisDecomp::decompose List adjncy; List xadj; - calcMetisCSR(globalCellCells, adjncy, xadj); + scotchDecomp::calcCSR(globalCellCells, adjncy, xadj); // Decompose using default weights diff --git a/src/parallel/decompositionMethods/metisDecomp/metisDecomp.H b/src/parallel/metisDecomp/metisDecomp.H similarity index 89% rename from src/parallel/decompositionMethods/metisDecomp/metisDecomp.H rename to src/parallel/metisDecomp/metisDecomp.H index 6a38e25c8a..c585686f60 100644 --- a/src/parallel/decompositionMethods/metisDecomp/metisDecomp.H +++ b/src/parallel/metisDecomp/metisDecomp.H @@ -132,24 +132,6 @@ public: const scalarField& cWeights ); - //- Helper to convert connectivity (supplied as owner,neighbour) into - // Metis storage - static void calcMetisCSR - ( - const polyMesh& mesh, - List& adjncy, - List& xadj - ); - - //- Helper to convert connectivity (supplied as cellcells) into - // Metis storage - static void calcMetisCSR - ( - const labelListList& globalCellCells, - List& adjncy, - List& xadj - ); - }; diff --git a/src/parallel/parMetisDecomp/Make/options b/src/parallel/parMetisDecomp/Make/options index 475bf5dbb1..79beb369af 100644 --- a/src/parallel/parMetisDecomp/Make/options +++ b/src/parallel/parMetisDecomp/Make/options @@ -4,7 +4,8 @@ EXE_INC = \ $(PFLAGS) $(PINC) \ -I$(WM_THIRD_PARTY_DIR)/ParMetis-3.1/ParMETISLib \ -I$(WM_THIRD_PARTY_DIR)/ParMetis-3.1 \ - -I../decompositionMethods/lnInclude + -I../decompositionMethods/lnInclude \ + -I../metisDecomp/lnInclude LIB_LIBS = \ -L$(FOAM_MPI_LIBBIN) \ diff --git a/src/parallel/parMetisDecomp/parMetisDecomp.C b/src/parallel/parMetisDecomp/parMetisDecomp.C index c4dde21d65..cf7434b5c9 100644 --- a/src/parallel/parMetisDecomp/parMetisDecomp.C +++ b/src/parallel/parMetisDecomp/parMetisDecomp.C @@ -26,6 +26,7 @@ License #include "parMetisDecomp.H" #include "metisDecomp.H" +#include "scotchDecomp.H" #include "syncTools.H" #include "addToRunTimeSelectionTable.H" #include "floatScalar.H" @@ -434,7 +435,7 @@ Foam::labelList Foam::parMetisDecomp::decompose { FatalErrorIn ( - "metisDecomp::decompose" + "parMetisDecomp::decompose" "(const pointField&, const scalarField&)" ) << "Number of cell weights " << cWeights.size() << " does not equal number of cells " << mesh_.nCells() @@ -762,7 +763,7 @@ Foam::labelList Foam::parMetisDecomp::decompose Field adjncy; // Offsets into adjncy Field xadj; - metisDecomp::calcMetisCSR(globalCellCells, adjncy, xadj); + scotchDecomp::calcCSR(globalCellCells, adjncy, xadj); // decomposition options. 0 = use defaults List options(3, 0);