Moved metis into its own library.

Scotch is now built using its own build system which builds
an additional metis.h which conflicts with the metis one so
to have separate include directory settings we also need separate
Make/ structures.

src/mesh/autoMesh/Make/options

@@ -8,7 +8,6 @@ EXE_INC = \
     -I$(LIB_SRC)/triSurface/lnInclude
 
 LIB_LIBS = \
-    -ldecompositionMethods \
     -ldynamicMesh \
     -lfiniteVolume \
     -llagrangian \

src/meshTools/Make/options

@@ -6,4 +6,5 @@ EXE_INC = \
 LIB_LIBS = \
     -ltriSurface \
     -ldecompositionMethods \
+    -lmetisDecompositionMethod \
     -llagrangian

src/parallel/Allwmake

@@ -3,6 +3,7 @@ cd ${0%/*} || exit 1    # run from this directory
 set -x
 
 wmake libso decompositionMethods
+wmake libso metisDecomp
 wmake libso reconstruct
 
 if [ -d "$FOAM_MPI_LIBBIN" ]

src/parallel/decompositionMethods/Make/files

@@ -6,6 +6,4 @@ manualDecomp/manualDecomp.C
 
 scotchDecomp/scotchDecomp.C
 
-metisDecomp/metisDecomp.C
-
 LIB = $(FOAM_LIBBIN)/libdecompositionMethods

src/parallel/decompositionMethods/Make/options

@@ -1,8 +1,6 @@
 EXE_INC = \
-    -I$(WM_THIRD_PARTY_DIR)/scotch_5.1/src/libscotch/lnInclude \
-    -I$(WM_THIRD_PARTY_DIR)/metis-5.0pre2/include
+    -I$(WM_THIRD_PARTY_DIR)/scotch_5.1/include \
+    -I../metisDecomp/lnInclude
 
 LIB_LIBS = \
-    -lscotch \
-    -lmetis \
-    -lGKlib
+    -lscotch -lscotcherrexit

src/parallel/decompositionMethods/scotchDecomp/scotchDecomp.C

@@ -61,17 +61,12 @@ License
 #include "scotchDecomp.H"
 #include "addToRunTimeSelectionTable.H"
 #include "floatScalar.H"
-#include "IFstream.H"
 #include "Time.H"
 #include "cyclicPolyPatch.H"
 #include "OFstream.H"
-#include "metisDecomp.H"
 
 extern "C"
 {
-#define OMPI_SKIP_MPICXX
-#include "module.h"
-#include "common.h"
 #include "scotch.h"
 }
 
@@ -115,13 +110,13 @@ void Foam::scotchDecomp::check(const int retVal, const char* str)
     if (retVal)
     {
         FatalErrorIn("scotchDecomp::decompose(..)")
-            << "Called to scotch routine " << str << " failed."
+            << "Call to scotch routine " << str << " failed."
             << exit(FatalError);
     }
 }
 
 
-// Call Metis with options from dictionary.
+// Call scotch with options from dictionary.
 Foam::label Foam::scotchDecomp::decompose
 (
     const List<int>& adjncy,
@@ -173,7 +168,7 @@ Foam::label Foam::scotchDecomp::decompose
         {
             FatalErrorIn
             (
-                "parMetisDecomp::decompose"
+                "scotchDecomp::decompose"
                 "(const pointField&, const scalarField&)"
             )   << "Number of cell weights " << cWeights.size()
                 << " does not equal number of cells " << xadj.size()-1
@@ -377,7 +372,10 @@ Foam::labelList Foam::scotchDecomp::decompose
 {
     if (points.size() != mesh_.nCells())
     {
-        FatalErrorIn("scotchDecomp::decompose(const pointField&)")
+        FatalErrorIn
+        (
+            "scotchDecomp::decompose(const pointField&, const scalarField&)"
+        )
             << "Can use this decomposition method only for the whole mesh"
             << endl
             << "and supply one coordinate (cellCentre) for every cell." << endl
@@ -391,12 +389,7 @@ Foam::labelList Foam::scotchDecomp::decompose
     //   xadj(celli) : start of information in adjncy for celli
     List<int> adjncy;
     List<int> xadj;
-    metisDecomp::calcMetisCSR
-    (
-        mesh_,
-        adjncy,
-        xadj
-    );
+    calcCSR(mesh_, adjncy, xadj);
 
     // Decompose using default weights
     List<int> finalDecomp;
@@ -445,7 +438,7 @@ Foam::labelList Foam::scotchDecomp::decompose
             cellCells
         );
 
-        metisDecomp::calcMetisCSR(cellCells, adjncy, xadj);
+        calcCSR(cellCells, adjncy, xadj);
     }
 
     // Decompose using weights
@@ -475,7 +468,8 @@ Foam::labelList Foam::scotchDecomp::decompose
     {
         FatalErrorIn
         (
-            "scotchDecomp::decompose(const pointField&, const labelListList&)"
+            "scotchDecomp::decompose"
+            "(const labelListList&, const pointField&, const scalarField&)"
         )   << "Inconsistent number of cells (" << globalCellCells.size()
             << ") and number of cell centres (" << cellCentres.size()
             << ")." << exit(FatalError);
@@ -488,7 +482,7 @@ Foam::labelList Foam::scotchDecomp::decompose
 
     List<int> adjncy;
     List<int> xadj;
-    metisDecomp::calcMetisCSR(globalCellCells, adjncy, xadj);
+    calcCSR(globalCellCells, adjncy, xadj);
 
     // Decompose using weights
     List<int> finalDecomp;
@@ -504,4 +498,144 @@ Foam::labelList Foam::scotchDecomp::decompose
 }
 
 
+void Foam::scotchDecomp::calcCSR
+(
+    const polyMesh& mesh,
+    List<int>& adjncy,
+    List<int>& xadj
+)
+{
+    // Make Metis CSR (Compressed Storage Format) storage
+    //   adjncy      : contains neighbours (= edges in graph)
+    //   xadj(celli) : start of information in adjncy for celli
+
+    xadj.setSize(mesh.nCells()+1);
+
+    // Initialise the number of internal faces of the cells to twice the
+    // number of internal faces
+    label nInternalFaces = 2*mesh.nInternalFaces();
+
+    // Check the boundary for coupled patches and add to the number of
+    // internal faces
+    const polyBoundaryMesh& pbm = mesh.boundaryMesh();
+
+    forAll(pbm, patchi)
+    {
+        if (isA<cyclicPolyPatch>(pbm[patchi]))
+        {
+            nInternalFaces += pbm[patchi].size();
+        }
+    }
+
+    // Create the adjncy array the size of the total number of internal and
+    // coupled faces
+    adjncy.setSize(nInternalFaces);
+
+    // Fill in xadj
+    // ~~~~~~~~~~~~
+    label freeAdj = 0;
+
+    for (label cellI = 0; cellI < mesh.nCells(); cellI++)
+    {
+        xadj[cellI] = freeAdj;
+
+        const labelList& cFaces = mesh.cells()[cellI];
+
+        forAll(cFaces, i)
+        {
+            label faceI = cFaces[i];
+
+            if
+            (
+                mesh.isInternalFace(faceI)
+             || isA<cyclicPolyPatch>(pbm[pbm.whichPatch(faceI)])
+            )
+            {
+                freeAdj++;
+            }
+        }
+    }
+    xadj[mesh.nCells()] = freeAdj;
+
+
+    // Fill in adjncy
+    // ~~~~~~~~~~~~~~
+
+    labelList nFacesPerCell(mesh.nCells(), 0);
+
+    // Internal faces
+    for (label faceI = 0; faceI < mesh.nInternalFaces(); faceI++)
+    {
+        label own = mesh.faceOwner()[faceI];
+        label nei = mesh.faceNeighbour()[faceI];
+
+        adjncy[xadj[own] + nFacesPerCell[own]++] = nei;
+        adjncy[xadj[nei] + nFacesPerCell[nei]++] = own;
+    }
+
+    // Coupled faces. Only cyclics done.
+    forAll(pbm, patchi)
+    {
+        if (isA<cyclicPolyPatch>(pbm[patchi]))
+        {
+            const unallocLabelList& faceCells = pbm[patchi].faceCells();
+
+            label sizeby2 = faceCells.size()/2;
+
+            for (label facei=0; facei<sizeby2; facei++)
+            {
+                label own = faceCells[facei];
+                label nei = faceCells[facei + sizeby2];
+
+                adjncy[xadj[own] + nFacesPerCell[own]++] = nei;
+                adjncy[xadj[nei] + nFacesPerCell[nei]++] = own;
+            }
+        }
+    }
+}
+
+
+// From cell-cell connections to Metis format (like CompactListList)
+void Foam::scotchDecomp::calcCSR
+(
+    const labelListList& cellCells,
+    List<int>& adjncy,
+    List<int>& xadj
+)
+{
+    // Count number of internal faces
+    label nConnections = 0;
+
+    forAll(cellCells, coarseI)
+    {
+        nConnections += cellCells[coarseI].size();
+    }
+
+    // Create the adjncy array as twice the size of the total number of
+    // internal faces
+    adjncy.setSize(nConnections);
+
+    xadj.setSize(cellCells.size()+1);
+
+
+    // Fill in xadj
+    // ~~~~~~~~~~~~
+    label freeAdj = 0;
+
+    forAll(cellCells, coarseI)
+    {
+        xadj[coarseI] = freeAdj;
+
+        const labelList& cCells = cellCells[coarseI];
+
+        forAll(cCells, i)
+        {
+            adjncy[freeAdj++] = cCells[i];
+        }
+    }
+    xadj[cellCells.size()] = freeAdj;
+}
+
+
+
 // ************************************************************************* //

src/parallel/decompositionMethods/scotchDecomp/scotchDecomp.H

@@ -135,6 +135,25 @@ public:
             const scalarField& cWeights
         );
 
+
+        //- Helper to convert local connectivity (supplied as owner,neighbour)
+        //  into CSR (Metis,scotch) storage. Does cyclics but not processor
+        //  patches
+        static void calcCSR
+        (
+            const polyMesh& mesh,
+            List<int>& adjncy,
+            List<int>& xadj
+        );
+
+        //- Helper to convert connectivity (supplied as cellcells) into
+        //  CSR (Metis,scotch) storage
+        static void calcCSR
+        (
+            const labelListList& globalCellCells,
+            List<int>& adjncy,
+            List<int>& xadj
+        );
 };
 
 

src/parallel/metisDecomp/Make/files

@@ -0,0 +1,3 @@
+metisDecomp.C
+
+LIB = $(FOAM_LIBBIN)/libmetisDecompositionMethod

src/parallel/metisDecomp/Make/options

@@ -0,0 +1,7 @@
+EXE_INC = \
+    -I$(WM_THIRD_PARTY_DIR)/metis-5.0pre2/include \
+    -I../decompositionMethods/lnInclude
+
+LIB_LIBS = \
+    -lmetis \
+    -lGKlib

src/parallel/metisDecomp/metisDecomp.C

@@ -27,9 +27,8 @@ License
 #include "metisDecomp.H"
 #include "addToRunTimeSelectionTable.H"
 #include "floatScalar.H"
-#include "IFstream.H"
 #include "Time.H"
-#include "cyclicPolyPatch.H"
+#include "scotchDecomp.H"
 
 extern "C"
 {
@@ -331,12 +330,7 @@ Foam::labelList Foam::metisDecomp::decompose
 
     List<int> adjncy;
     List<int> xadj;
-    calcMetisCSR
-    (
-        mesh_,
-        adjncy,
-        xadj
-    );
+    scotchDecomp::calcCSR(mesh_, adjncy, xadj);
 
     // Decompose using default weights
     List<int> finalDecomp;
@@ -352,145 +346,6 @@ Foam::labelList Foam::metisDecomp::decompose
 }
 
 
-void Foam::metisDecomp::calcMetisCSR
-(
-    const polyMesh& mesh,
-    List<int>& adjncy,
-    List<int>& xadj
-)
-{
-    // Make Metis CSR (Compressed Storage Format) storage
-    //   adjncy      : contains neighbours (= edges in graph)
-    //   xadj(celli) : start of information in adjncy for celli
-
-    xadj.setSize(mesh.nCells()+1);
-
-    // Initialise the number of internal faces of the cells to twice the
-    // number of internal faces
-    label nInternalFaces = 2*mesh.nInternalFaces();
-
-    // Check the boundary for coupled patches and add to the number of
-    // internal faces
-    const polyBoundaryMesh& pbm = mesh.boundaryMesh();
-
-    forAll(pbm, patchi)
-    {
-        if (isA<cyclicPolyPatch>(pbm[patchi]))
-        {
-            nInternalFaces += pbm[patchi].size();
-        }
-    }
-
-    // Create the adjncy array the size of the total number of internal and
-    // coupled faces
-    adjncy.setSize(nInternalFaces);
-
-    // Fill in xadj
-    // ~~~~~~~~~~~~
-    label freeAdj = 0;
-
-    for (label cellI = 0; cellI < mesh.nCells(); cellI++)
-    {
-        xadj[cellI] = freeAdj;
-
-        const labelList& cFaces = mesh.cells()[cellI];
-
-        forAll(cFaces, i)
-        {
-            label faceI = cFaces[i];
-
-            if
-            (
-                mesh.isInternalFace(faceI)
-             || isA<cyclicPolyPatch>(pbm[pbm.whichPatch(faceI)])
-            )
-            {
-                freeAdj++;
-            }
-        }
-    }
-    xadj[mesh.nCells()] = freeAdj;
-
-
-    // Fill in adjncy
-    // ~~~~~~~~~~~~~~
-
-    labelList nFacesPerCell(mesh.nCells(), 0);
-
-    // Internal faces
-    for (label faceI = 0; faceI < mesh.nInternalFaces(); faceI++)
-    {
-        label own = mesh.faceOwner()[faceI];
-        label nei = mesh.faceNeighbour()[faceI];
-
-        adjncy[xadj[own] + nFacesPerCell[own]++] = nei;
-        adjncy[xadj[nei] + nFacesPerCell[nei]++] = own;
-    }
-
-    // Coupled faces. Only cyclics done.
-    forAll(pbm, patchi)
-    {
-        if (isA<cyclicPolyPatch>(pbm[patchi]))
-        {
-            const unallocLabelList& faceCells = pbm[patchi].faceCells();
-
-            label sizeby2 = faceCells.size()/2;
-
-            for (label facei=0; facei<sizeby2; facei++)
-            {
-                label own = faceCells[facei];
-                label nei = faceCells[facei + sizeby2];
-
-                adjncy[xadj[own] + nFacesPerCell[own]++] = nei;
-                adjncy[xadj[nei] + nFacesPerCell[nei]++] = own;
-            }
-        }
-    }
-}
-
-
-// From cell-cell connections to Metis format (like CompactListList)
-void Foam::metisDecomp::calcMetisCSR
-(
-    const labelListList& cellCells,
-    List<int>& adjncy,
-    List<int>& xadj
-)
-{
-    // Count number of internal faces
-    label nConnections = 0;
-
-    forAll(cellCells, coarseI)
-    {
-        nConnections += cellCells[coarseI].size();
-    }
-
-    // Create the adjncy array as twice the size of the total number of
-    // internal faces
-    adjncy.setSize(nConnections);
-
-    xadj.setSize(cellCells.size()+1);
-
-
-    // Fill in xadj
-    // ~~~~~~~~~~~~
-    label freeAdj = 0;
-
-    forAll(cellCells, coarseI)
-    {
-        xadj[coarseI] = freeAdj;
-
-        const labelList& cCells = cellCells[coarseI];
-
-        forAll(cCells, i)
-        {
-            adjncy[freeAdj++] = cCells[i];
-        }
-    }
-    xadj[cellCells.size()] = freeAdj;
-}
-
-
 Foam::labelList Foam::metisDecomp::decompose
 (
     const labelList& agglom,
@@ -525,7 +380,7 @@ Foam::labelList Foam::metisDecomp::decompose
             cellCells
         );
 
-        calcMetisCSR(cellCells, adjncy, xadj);
+        scotchDecomp::calcCSR(cellCells, adjncy, xadj);
     }
 
     // Decompose using default weights
@@ -570,7 +425,7 @@ Foam::labelList Foam::metisDecomp::decompose
 
     List<int> adjncy;
     List<int> xadj;
-    calcMetisCSR(globalCellCells, adjncy, xadj);
+    scotchDecomp::calcCSR(globalCellCells, adjncy, xadj);
 
 
     // Decompose using default weights

src/parallel/metisDecomp/metisDecomp.H

@@ -132,24 +132,6 @@ public:
             const scalarField& cWeights
         );
 
-        //- Helper to convert connectivity (supplied as owner,neighbour) into
-        //  Metis storage
-        static void calcMetisCSR
-        (
-            const polyMesh& mesh,
-            List<int>& adjncy,
-            List<int>& xadj
-        );
-
-        //- Helper to convert connectivity (supplied as cellcells) into
-        //  Metis storage
-        static void calcMetisCSR
-        (
-            const labelListList& globalCellCells,
-            List<int>& adjncy,
-            List<int>& xadj
-        );
-
 };
 
 

src/parallel/parMetisDecomp/Make/options

@@ -4,7 +4,8 @@ EXE_INC = \
     $(PFLAGS) $(PINC) \
     -I$(WM_THIRD_PARTY_DIR)/ParMetis-3.1/ParMETISLib \
     -I$(WM_THIRD_PARTY_DIR)/ParMetis-3.1 \
-    -I../decompositionMethods/lnInclude
+    -I../decompositionMethods/lnInclude \
+    -I../metisDecomp/lnInclude
 
 LIB_LIBS = \
     -L$(FOAM_MPI_LIBBIN) \

src/parallel/parMetisDecomp/parMetisDecomp.C

@@ -26,6 +26,7 @@ License
 
 #include "parMetisDecomp.H"
 #include "metisDecomp.H"
+#include "scotchDecomp.H"
 #include "syncTools.H"
 #include "addToRunTimeSelectionTable.H"
 #include "floatScalar.H"
@@ -434,7 +435,7 @@ Foam::labelList Foam::parMetisDecomp::decompose
         {
             FatalErrorIn
             (
-                "metisDecomp::decompose"
+                "parMetisDecomp::decompose"
                 "(const pointField&, const scalarField&)"
             )   << "Number of cell weights " << cWeights.size()
                 << " does not equal number of cells " << mesh_.nCells()
@@ -762,7 +763,7 @@ Foam::labelList Foam::parMetisDecomp::decompose
     Field<int> adjncy;
     // Offsets into adjncy
     Field<int> xadj;
-    metisDecomp::calcMetisCSR(globalCellCells, adjncy, xadj);
+    scotchDecomp::calcCSR(globalCellCells, adjncy, xadj);
 
     // decomposition options. 0 = use defaults
     List<int> options(3, 0);