From 2c48a69e505d5e11317435359ca49f6d65f3958f Mon Sep 17 00:00:00 2001 From: mattijs Date: Thu, 23 Apr 2009 14:44:23 +0100 Subject: [PATCH] scotch decomposition method --- .../decompositionMethods/Make/files | 5 +- .../decompositionMethods/Make/options | 2 + .../scotchDecomp/scotchDecomp.C | 425 ++++++++++++++++++ .../scotchDecomp/scotchDecomp.H | 150 +++++++ wmake/rules/General/bison | 4 +- wmake/rules/General/standard | 5 +- 6 files changed, 586 insertions(+), 5 deletions(-) create mode 100644 src/decompositionMethods/decompositionMethods/scotchDecomp/scotchDecomp.C create mode 100644 src/decompositionMethods/decompositionMethods/scotchDecomp/scotchDecomp.H diff --git a/src/decompositionMethods/decompositionMethods/Make/files b/src/decompositionMethods/decompositionMethods/Make/files index 9707bc7f2e..2acbfd5606 100644 --- a/src/decompositionMethods/decompositionMethods/Make/files +++ b/src/decompositionMethods/decompositionMethods/Make/files @@ -2,7 +2,10 @@ decompositionMethod/decompositionMethod.C geomDecomp/geomDecomp.C simpleGeomDecomp/simpleGeomDecomp.C hierarchGeomDecomp/hierarchGeomDecomp.C -metisDecomp/metisDecomp.C manualDecomp/manualDecomp.C +scotchDecomp/scotchDecomp.C + +metisDecomp/metisDecomp.C + LIB = $(FOAM_LIBBIN)/libdecompositionMethods diff --git a/src/decompositionMethods/decompositionMethods/Make/options b/src/decompositionMethods/decompositionMethods/Make/options index 9ccb652d2b..97569aa663 100644 --- a/src/decompositionMethods/decompositionMethods/Make/options +++ b/src/decompositionMethods/decompositionMethods/Make/options @@ -1,6 +1,8 @@ EXE_INC = \ + -I$(WM_THIRD_PARTY_DIR)/scotch_5.1/src/libscotch/lnInclude \ -I$(WM_THIRD_PARTY_DIR)/metis-5.0pre2/include LIB_LIBS = \ + -lscotch \ -lmetis \ -lGKlib diff --git a/src/decompositionMethods/decompositionMethods/scotchDecomp/scotchDecomp.C b/src/decompositionMethods/decompositionMethods/scotchDecomp/scotchDecomp.C new file mode 100644 index 0000000000..2746d8f4e9 --- /dev/null +++ b/src/decompositionMethods/decompositionMethods/scotchDecomp/scotchDecomp.C @@ -0,0 +1,425 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | OpenFOAM: The Open Source CFD Toolbox + \\ / O peration | + \\ / A nd | Copyright (C) 1991-2009 OpenCFD Ltd. + \\/ M anipulation | +------------------------------------------------------------------------------- +License + This file is part of OpenFOAM. + + OpenFOAM is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the + Free Software Foundation; either version 2 of the License, or (at your + option) any later version. + + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with OpenFOAM; if not, write to the Free Software Foundation, + Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +S Strat=b{job=t,map=t,poli=S,sep=(m{asc=b{bnd=d{pass=40,dif=1,rem=1}f{move=80,pass=-1,bal=0.005},org=f{move=80,pass=-1,bal=0.005},width=3},low=h{pass=10}f{move=80,pass=-1,bal=0.0005},type=h,vert=80,rat=0.8}|m{asc=b{bnd=d{pass=40,dif=1,rem=1}f{move=80,pass=-1,bal=0.005},org=f{move=80,pass=-1,bal=0.005},width=3},low=h{pass=10}f{move=80,pass=-1,bal=0.0005},type=h,vert=80,rat=0.8})} +\*---------------------------------------------------------------------------*/ + +#include "scotchDecomp.H" +#include "addToRunTimeSelectionTable.H" +#include "floatScalar.H" +#include "IFstream.H" +#include "Time.H" +#include "cyclicPolyPatch.H" + +extern "C" +{ +#define OMPI_SKIP_MPICXX +#include "module.h" +#include "common.h" +#include "scotch.h" +} + + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +namespace Foam +{ + defineTypeNameAndDebug(scotchDecomp, 0); + + addToRunTimeSelectionTable + ( + decompositionMethod, + scotchDecomp, + dictionaryMesh + ); +} + +// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * // + +void Foam::scotchDecomp::check(const int retVal, const char* str) +{ + if (retVal) + { + FatalErrorIn("scotchDecomp::decompose(..)") + << "Called to scotch routine " << str << " failed." + << exit(FatalError); + } +} + + +// Call Metis with options from dictionary. +Foam::label Foam::scotchDecomp::decompose +( + const List& adjncy, + const List& xadj, + + List& finalDecomp +) +{ + // Strategy + // ~~~~~~~~ + // Default. + SCOTCH_Strat stradat; + check(SCOTCH_stratInit(&stradat), "SCOTCH_stratInit"); + //SCOTCH_stratGraphMap(&stradat, &argv[i][2]); + //fprintf(stdout, "S\tStrat="); + //SCOTCH_stratSave(&stradat, stdout); + //fprintf(stdout, "\n"); + + + // Graph + // ~~~~~ + + SCOTCH_Graph grafdat; + check(SCOTCH_graphInit(&grafdat), "SCOTCH_graphInit"); + check + ( + SCOTCH_graphBuild + ( + &grafdat, + 0, // baseval, c-style numbering + xadj.size()-1, // vertnbr, nCells + xadj.begin(), // verttab, start index per cell into adjncy + &xadj[1], // vendtab, end index ,, + NULL, // velotab, vertex weights + NULL, // vlbltab + adjncy.size(), // edgenbr, number of arcs + adjncy.begin(), // edgetab + NULL // edlotab, edge weights + ), + "SCOTCH_graphBuild" + ); + check(SCOTCH_graphCheck(&grafdat), "SCOTCH_graphCheck"); + + + + //// Architecture + //// ~~~~~~~~~~~~ + //// (fully connected network topology since using switch) + // + //SCOTCH_Arch archdat; + //check(SCOTCH_archInit(&archdat), "SCOTCH_archInit"); + //check + //( + // // SCOTCH_archCmpltw for weighted. + // SCOTCH_archCmplt(&archdat, nProcessors_), + // "SCOTCH_archCmplt" + //); + + + + + //SCOTCH_Mapping mapdat; + //SCOTCH_graphMapInit(&grafdat, &mapdat, &archdat, NULL); + //SCOTCH_graphMapCompute(&grafdat, &mapdat, &stradat); /* Perform mapping */ + + + finalDecomp.setSize(xadj.size()-1); + check + ( + SCOTCH_graphPart + ( + &grafdat, + nProcessors_, // partnbr + &stradat, // const SCOTCH_Strat * + finalDecomp.begin() // parttab + ), + "SCOTCH_graphPart" + ); + + // Release storage for graph + SCOTCH_graphExit(&grafdat); + SCOTCH_stratExit(&stradat); + //// Release storage for network topology + //SCOTCH_archExit(&archdat); + + return 0; +} + + +// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * // + +Foam::scotchDecomp::scotchDecomp +( + const dictionary& decompositionDict, + const polyMesh& mesh +) +: + decompositionMethod(decompositionDict), + mesh_(mesh) +{} + + +// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // + +Foam::labelList Foam::scotchDecomp::decompose(const pointField& points) +{ + if (points.size() != mesh_.nCells()) + { + FatalErrorIn("scotchDecomp::decompose(const pointField&)") + << "Can use this decomposition method only for the whole mesh" + << endl + << "and supply one coordinate (cellCentre) for every cell." << endl + << "The number of coordinates " << points.size() << endl + << "The number of cells in the mesh " << mesh_.nCells() + << exit(FatalError); + } + + // Make Metis CSR (Compressed Storage Format) storage + // adjncy : contains neighbours (= edges in graph) + // xadj(celli) : start of information in adjncy for celli + + List xadj(mesh_.nCells()+1); + + // Initialise the number of internal faces of the cells to twice the + // number of internal faces + label nInternalFaces = 2*mesh_.nInternalFaces(); + + // Check the boundary for coupled patches and add to the number of + // internal faces + const polyBoundaryMesh& pbm = mesh_.boundaryMesh(); + + forAll(pbm, patchi) + { + if (isA(pbm[patchi])) + { + nInternalFaces += pbm[patchi].size(); + } + } + + // Create the adjncy array the size of the total number of internal and + // coupled faces + List adjncy(nInternalFaces); + + // Fill in xadj + // ~~~~~~~~~~~~ + label freeAdj = 0; + + for (label cellI = 0; cellI < mesh_.nCells(); cellI++) + { + xadj[cellI] = freeAdj; + + const labelList& cFaces = mesh_.cells()[cellI]; + + forAll(cFaces, i) + { + label faceI = cFaces[i]; + + if + ( + mesh_.isInternalFace(faceI) + || isA(pbm[pbm.whichPatch(faceI)]) + ) + { + freeAdj++; + } + } + } + xadj[mesh_.nCells()] = freeAdj; + + + // Fill in adjncy + // ~~~~~~~~~~~~~~ + + labelList nFacesPerCell(mesh_.nCells(), 0); + + // Internal faces + for (label faceI = 0; faceI < mesh_.nInternalFaces(); faceI++) + { + label own = mesh_.faceOwner()[faceI]; + label nei = mesh_.faceNeighbour()[faceI]; + + adjncy[xadj[own] + nFacesPerCell[own]++] = nei; + adjncy[xadj[nei] + nFacesPerCell[nei]++] = own; + } + + // Coupled faces. Only cyclics done. + forAll(pbm, patchi) + { + if (isA(pbm[patchi])) + { + const unallocLabelList& faceCells = pbm[patchi].faceCells(); + + label sizeby2 = faceCells.size()/2; + + for (label facei=0; facei finalDecomp; + decompose(adjncy, xadj, finalDecomp); + + // Copy back to labelList + labelList decomp(finalDecomp.size()); + forAll(decomp, i) + { + decomp[i] = finalDecomp[i]; + } + return decomp; +} + + +// From cell-cell connections to Metis format (like CompactListList) +void Foam::scotchDecomp::calcMetisCSR +( + const labelListList& cellCells, + List& adjncy, + List& xadj +) +{ + // Count number of internal faces + label nConnections = 0; + + forAll(cellCells, coarseI) + { + nConnections += cellCells[coarseI].size(); + } + + // Create the adjncy array as twice the size of the total number of + // internal faces + adjncy.setSize(nConnections); + + xadj.setSize(cellCells.size()+1); + + + // Fill in xadj + // ~~~~~~~~~~~~ + label freeAdj = 0; + + forAll(cellCells, coarseI) + { + xadj[coarseI] = freeAdj; + + const labelList& cCells = cellCells[coarseI]; + + forAll(cCells, i) + { + adjncy[freeAdj++] = cCells[i]; + } + } + xadj[cellCells.size()] = freeAdj; +} + + +Foam::labelList Foam::scotchDecomp::decompose +( + const labelList& agglom, + const pointField& agglomPoints +) +{ + if (agglom.size() != mesh_.nCells()) + { + FatalErrorIn + ( + "parMetisDecomp::decompose(const labelList&, const pointField&)" + ) << "Size of cell-to-coarse map " << agglom.size() + << " differs from number of cells in mesh " << mesh_.nCells() + << exit(FatalError); + } + + // Make Metis CSR (Compressed Storage Format) storage + // adjncy : contains neighbours (= edges in graph) + // xadj(celli) : start of information in adjncy for celli + List adjncy; + List xadj; + { + // Get cellCells on coarse mesh. + labelListList cellCells; + calcCellCells + ( + mesh_, + agglom, + agglomPoints.size(), + cellCells + ); + + calcMetisCSR(cellCells, adjncy, xadj); + } + + // Decompose using default weights + List finalDecomp; + decompose(adjncy, xadj, finalDecomp); + + + // Rework back into decomposition for original mesh_ + labelList fineDistribution(agglom.size()); + + forAll(fineDistribution, i) + { + fineDistribution[i] = finalDecomp[agglom[i]]; + } + + return fineDistribution; +} + + +Foam::labelList Foam::scotchDecomp::decompose +( + const labelListList& globalCellCells, + const pointField& cellCentres +) +{ + if (cellCentres.size() != globalCellCells.size()) + { + FatalErrorIn + ( + "scotchDecomp::decompose(const pointField&, const labelListList&)" + ) << "Inconsistent number of cells (" << globalCellCells.size() + << ") and number of cell centres (" << cellCentres.size() + << ")." << exit(FatalError); + } + + + // Make Metis CSR (Compressed Storage Format) storage + // adjncy : contains neighbours (= edges in graph) + // xadj(celli) : start of information in adjncy for celli + + List adjncy; + List xadj; + calcMetisCSR(globalCellCells, adjncy, xadj); + + + // Decompose using default weights + List finalDecomp; + decompose(adjncy, xadj, finalDecomp); + + // Copy back to labelList + labelList decomp(finalDecomp.size()); + forAll(decomp, i) + { + decomp[i] = finalDecomp[i]; + } + return decomp; +} + + +// ************************************************************************* // diff --git a/src/decompositionMethods/decompositionMethods/scotchDecomp/scotchDecomp.H b/src/decompositionMethods/decompositionMethods/scotchDecomp/scotchDecomp.H new file mode 100644 index 0000000000..e470f84f98 --- /dev/null +++ b/src/decompositionMethods/decompositionMethods/scotchDecomp/scotchDecomp.H @@ -0,0 +1,150 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | OpenFOAM: The Open Source CFD Toolbox + \\ / O peration | + \\ / A nd | Copyright (C) 1991-2009 OpenCFD Ltd. + \\/ M anipulation | +------------------------------------------------------------------------------- +License + This file is part of OpenFOAM. + + OpenFOAM is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the + Free Software Foundation; either version 2 of the License, or (at your + option) any later version. + + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with OpenFOAM; if not, write to the Free Software Foundation, + Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +Class + Foam::scotchDecomp + +Description + Scotch domain decomposition + +SourceFiles + scotchDecomp.C + +\*---------------------------------------------------------------------------*/ + +#ifndef scotchDecomp_H +#define scotchDecomp_H + +#include "decompositionMethod.H" + +namespace Foam +{ + +/*---------------------------------------------------------------------------*\ + Class scotchDecomp Declaration +\*---------------------------------------------------------------------------*/ + +class scotchDecomp +: + public decompositionMethod +{ + // Private data + + const polyMesh& mesh_; + + + // Private Member Functions + + //- Check and print error message + static void check(const int, const char*); + + label decompose + ( + const List& adjncy, + const List& xadj, + List& finalDecomp + ); + + //- Disallow default bitwise copy construct and assignment + void operator=(const scotchDecomp&); + scotchDecomp(const scotchDecomp&); + + +public: + + //- Runtime type information + TypeName("scotch"); + + + // Constructors + + //- Construct given the decomposition dictionary and mesh + scotchDecomp + ( + const dictionary& decompositionDict, + const polyMesh& mesh + ); + + + // Destructor + + virtual ~scotchDecomp() + {} + + + // Member Functions + + virtual bool parallelAware() const + { + // Metis does not know about proc boundaries + return false; + } + + //- Return for every coordinate the wanted processor number. Use the + // mesh connectivity (if needed) + virtual labelList decompose(const pointField&); + + //- Return for every coordinate the wanted processor number. Gets + // passed agglomeration map (from fine to coarse cells) and coarse cell + // location. Can be overridden by decomposers that provide this + // functionality natively. + virtual labelList decompose + ( + const labelList& agglom, + const pointField& + ); + + //- Return for every coordinate the wanted processor number. Explicitly + // provided mesh connectivity. + // The connectivity is equal to mesh.cellCells() except for + // - in parallel the cell numbers are global cell numbers (starting + // from 0 at processor0 and then incrementing all through the + // processors) + // - the connections are across coupled patches + virtual labelList decompose + ( + const labelListList& globalCellCells, + const pointField& cc + ); + + //- Helper to convert cellcells into compact row storage + static void calcMetisCSR + ( + const labelListList& globalCellCells, + List& adjncy, + List& xadj + ); + +}; + + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +} // End namespace Foam + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +#endif + +// ************************************************************************* // diff --git a/wmake/rules/General/bison b/wmake/rules/General/bison index 4c63c1f565..fe2e00a097 100644 --- a/wmake/rules/General/bison +++ b/wmake/rules/General/bison @@ -1,8 +1,8 @@ .SUFFIXES: .y .Y -ytoo = bison -v $(YYPREFIX) -d $$SOURCE ; mv $$SOURCE_DIR/*.tab.c $*.c ; mv $$SOURCE_DIR/*.tab.h $*.h ; $(cc) $(cFLAGS) -c $*.c -o $@ +ytoo = bison -v -d -y $$SOURCE ; mv y.tab.c $*.c ; mv y.tab.h $*.h ; $(cc) $(cFLAGS) -c $*.c -o $@ -Ytoo = bison -v $(YYPREFIX) -d $$SOURCE ; mv $$SOURCE_DIR/*.tab.c $*.C ; mv $$SOURCE_DIR/*.tab.h $*.H ; $(CC) $(c++FLAGS) -c $*.C -o $@ +Ytoo = bison -v -d -y $$SOURCE ; mv y.tab.c $*.C ; mv y.tab.h $*.H ; $(CC) $(c++FLAGS) -c $*.C -o $@ .y.dep: $(MAKE_DEP) diff --git a/wmake/rules/General/standard b/wmake/rules/General/standard index 8e5e436d32..7752bef084 100644 --- a/wmake/rules/General/standard +++ b/wmake/rules/General/standard @@ -5,6 +5,7 @@ include $(GENERAL_RULES)/sourceToDep include $(GENERAL_RULES)/java include $(GENERAL_RULES)/flex include $(GENERAL_RULES)/flex++ -include $(GENERAL_RULES)/byacc -include $(GENERAL_RULES)/btyacc++ +#include $(GENERAL_RULES)/byacc +#include $(GENERAL_RULES)/btyacc++ +include $(GENERAL_RULES)/bison include $(GENERAL_RULES)/moc