Merge branch 'feature-kahip-decompose' into 'develop'

add support for KaHIP decomposition

See merge request Development/OpenFOAM-plus!154

applications/test/checkDecomposePar/Make/files

@@ -0,0 +1,3 @@
+Test-checkDecomposePar.C
+
+EXE = $(FOAM_APPBIN)/Test-checkDecomposePar

applications/test/checkDecomposePar/Make/options

@@ -0,0 +1,10 @@
+EXE_INC = \
+    -I$(LIB_SRC)/parallel/decompose/decompose/lnInclude \
+    -I$(LIB_SRC)/parallel/decompose/decompositionMethods/lnInclude \
+    -I$(LIB_SRC)/finiteVolume/lnInclude \
+    -I$(LIB_SRC)/meshTools/lnInclude \
+    -I$(LIB_SRC)/regionModels/regionModel/lnInclude
+
+EXE_LIBS = \
+    -ldecompositionMethods \
+    -lregionModels

applications/test/checkDecomposePar/Test-checkDecomposePar.C

@@ -0,0 +1,193 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | Copyright (C) 2017 OpenCFD Ltd.
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+Application
+    checkDecomposePar
+
+Group
+    grpParallelUtilities
+
+Description
+    Check decomposition from kaffpa (KaHIP) output.
+    foamToMetisGraph was likely used for producing the kaffpa input.
+
+\*---------------------------------------------------------------------------*/
+
+#include "OSspecific.H"
+#include "fvCFD.H"
+#include "cpuTime.H"
+#include "IFstream.H"
+#include "regionProperties.H"
+#include "decompositionInformation.H"
+#include "decompositionModel.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+int main(int argc, char *argv[])
+{
+    argList::addNote
+    (
+        "Check decomposition from kaffpa (KaHIP) output"
+    );
+
+    argList::noParallel();
+    argList::noBanner();
+
+    #include "addRegionOption.H"
+    argList::addBoolOption
+    (
+        "allRegions",
+        "operate on all regions in regionProperties"
+    );
+    argList::addBoolOption
+    (
+        "verbose",
+        "more information about decomposition"
+    );
+
+    argList::validArgs.append("kaffpa-output-file");
+
+    // Include explicit constant options, have zero from time range
+    timeSelector::addOptions(true, false);
+
+    #include "setRootCase.H"
+
+    const fileName decompFile = args[1];
+
+    const bool region     = args.optionFound("region");
+    const bool allRegions = args.optionFound("allRegions");
+    const bool verbose    = args.optionFound("verbose");
+
+    // Set time from database
+    #include "createTime.H"
+    // Allow override of time
+    instantList times = timeSelector::selectIfPresent(runTime, args);
+
+    // Allow override of decomposeParDict location
+    fileName decompDictFile;
+    args.optionReadIfPresent("decomposeParDict", decompDictFile);
+
+    wordList regionNames;
+    wordList regionDirs;
+    if (allRegions)
+    {
+        Info<< "Decomposing all regions in regionProperties" << nl << endl;
+        regionProperties rp(runTime);
+        forAllConstIters(rp, iter)
+        {
+            const wordList& regions = iter();
+            forAll(regions, i)
+            {
+                if (findIndex(regionNames, regions[i]) == -1)
+                {
+                    regionNames.append(regions[i]);
+                }
+            }
+        }
+        regionDirs = regionNames;
+    }
+    else
+    {
+        word regionName;
+        if (args.optionReadIfPresent("region", regionName))
+        {
+            regionNames = wordList(1, regionName);
+            regionDirs = regionNames;
+        }
+        else
+        {
+            regionNames = wordList(1, fvMesh::defaultRegion);
+            regionDirs = wordList(1, word::null);
+        }
+    }
+
+    labelList cellToProc;
+
+    forAll(regionNames, regioni)
+    {
+        const word& regionName = regionNames[regioni];
+        const word& regionDir = regionDirs[regioni];
+
+        Info<< "\n\nDecomposing mesh " << regionName << nl << endl;
+        Info<< "Create mesh..." << flush;
+
+        fvMesh mesh
+        (
+            IOobject
+            (
+                regionName,
+                runTime.timeName(),
+                runTime,
+                IOobject::MUST_READ,
+                IOobject::NO_WRITE,
+                false
+            )
+        );
+
+        Info<< " nCells = " << mesh.nCells() << endl;
+
+        // Expected format is a simple ASCII list
+        cellToProc.setSize(mesh.nCells());
+        {
+            IFstream is(decompFile);
+
+            forAll(cellToProc, celli)
+            {
+                cellToProc[celli] = readLabel(is);
+            }
+        }
+
+        const label nDomains = max(cellToProc) + 1;
+
+        CompactListList<label> cellCells;
+        decompositionMethod::calcCellCells
+        (
+            mesh,
+            identity(mesh.nCells()),
+            mesh.nCells(),
+            false,
+            cellCells
+        );
+
+        decompositionInformation info
+        (
+            cellCells,
+            cellToProc,
+            nDomains
+        );
+
+        if (verbose)
+        {
+            info.printDetails(Info);
+            Info<< nl;
+        }
+        info.printSummary(Info);
+    }
+
+    Info<< "\nEnd\n" << endl;
+
+    return 0;
+}
+
+
+// ************************************************************************* //

applications/test/decomposePar/Make/files

@@ -0,0 +1,3 @@
+Test-decomposePar.C
+
+EXE = $(FOAM_APPBIN)/Test-decomposePar

applications/test/decomposePar/Make/options

@@ -0,0 +1,12 @@
+EXE_INC = \
+    -I$(LIB_SRC)/parallel/decompose/decompose/lnInclude \
+    -I$(LIB_SRC)/parallel/decompose/decompositionMethods/lnInclude \
+    -I$(LIB_SRC)/finiteVolume/lnInclude \
+    -I$(LIB_SRC)/meshTools/lnInclude \
+    -I$(LIB_SRC)/regionModels/regionModel/lnInclude
+
+EXE_LIBS = \
+    -ldecompose \
+    -ldecompositionMethods \
+    -L$(FOAM_LIBBIN)/dummy -lmetisDecomp -lscotchDecomp -lkahipDecomp \
+    -lregionModels

applications/test/decomposePar/Test-decomposePar.C

@@ -0,0 +1,278 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | Copyright (C) 2017 OpenCFD Ltd.
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+Application
+    decomposePar
+
+Group
+    grpParallelUtilities
+
+Description
+    Automatically decomposes a mesh and fields of a case for parallel
+    execution of OpenFOAM.
+
+Usage
+    \b decomposePar [OPTION]
+
+    Options:
+      - \par -region \<regionName\>
+        Decompose named region. Does not check for existence of processor*.
+
+      - \par -allRegions
+        Decompose all regions in regionProperties. Does not check for
+        existence of processor*.
+
+      - \par -constant
+
+      - \par -time xxx:yyy
+        Override controlDict settings and decompose selected times. Does not
+        re-decompose the mesh i.e. does not handle moving mesh or changing
+        mesh cases.
+
+\*---------------------------------------------------------------------------*/
+
+#include "OSspecific.H"
+#include "fvCFD.H"
+#include "cpuTime.H"
+#include "IOobjectList.H"
+#include "regionProperties.H"
+#include "decompositionInformation.H"
+#include "decompositionModel.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+int main(int argc, char *argv[])
+{
+    argList::addNote
+    (
+        "decompose a mesh and fields of a case for parallel execution"
+    );
+
+    argList::noParallel();
+    argList::addOption
+    (
+        "decomposeParDict",
+        "file",
+        "read decomposePar dictionary from specified location"
+    );
+    #include "addRegionOption.H"
+    argList::addBoolOption
+    (
+        "allRegions",
+        "operate on all regions in regionProperties"
+    );
+    argList::addBoolOption
+    (
+        "verbose",
+        "more information about decomposition"
+    );
+    argList::addOption
+    (
+        "domains",
+        "N"
+        "override numberOfSubdomains"
+    );
+
+    argList::addOption
+    (
+        "method",
+        "name"
+        "override method"
+    );
+
+    // Include explicit constant options, have zero from time range
+    timeSelector::addOptions(true, false);
+
+    #include "setRootCase.H"
+
+    const bool region     = args.optionFound("region");
+    const bool allRegions = args.optionFound("allRegions");
+    const bool verbose    = args.optionFound("verbose");
+
+    const label numSubdomains =
+        args.optionLookupOrDefault<label>("domains", 0);
+
+    const word methodName =
+        args.optionLookupOrDefault<word>("method", word::null);
+
+
+    // Set time from database
+    #include "createTime.H"
+    // Allow override of time
+    instantList times = timeSelector::selectIfPresent(runTime, args);
+
+    // Allow override of decomposeParDict location
+    fileName decompDictFile;
+    args.optionReadIfPresent("decomposeParDict", decompDictFile);
+
+    wordList regionNames;
+    wordList regionDirs;
+    if (allRegions)
+    {
+        Info<< "Decomposing all regions in regionProperties" << nl << endl;
+        regionProperties rp(runTime);
+        forAllConstIters(rp, iter)
+        {
+            const wordList& regions = iter();
+            forAll(regions, i)
+            {
+                if (findIndex(regionNames, regions[i]) == -1)
+                {
+                    regionNames.append(regions[i]);
+                }
+            }
+        }
+        regionDirs = regionNames;
+    }
+    else
+    {
+        word regionName;
+        if (args.optionReadIfPresent("region", regionName))
+        {
+            regionNames = wordList(1, regionName);
+            regionDirs = regionNames;
+        }
+        else
+        {
+            regionNames = wordList(1, fvMesh::defaultRegion);
+            regionDirs = wordList(1, word::null);
+        }
+    }
+
+
+    forAll(regionNames, regioni)
+    {
+        const word& regionName = regionNames[regioni];
+        const word& regionDir = regionDirs[regioni];
+
+        Info<< "\n\nDecomposing mesh " << regionName << nl << endl;
+        Info<< "Create mesh..." << flush;
+
+        fvMesh mesh
+        (
+            IOobject
+            (
+                regionName,
+                runTime.timeName(),
+                runTime,
+                IOobject::MUST_READ,
+                IOobject::NO_WRITE,
+                false
+            )
+        );
+
+        Info<< " nCells = " << mesh.nCells() << endl;
+
+        Info<< "\nCalculating distribution of cells" << endl;
+        cpuTime decompositionTime;
+
+        const decompositionModel& model = decompositionModel::New
+        (
+            mesh,
+            decompDictFile
+        );
+
+        // Allow command-line override for quick testing
+
+        dictionary& modelDict = const_cast<decompositionModel&>(model);
+
+        if (numSubdomains)
+        {
+            modelDict.add
+            (
+                word("numberOfSubdomains"),
+                numSubdomains,
+                true
+            );
+        }
+
+        if (!methodName.empty())
+        {
+            modelDict.add
+            (
+                word("method"),
+                methodName,
+                true
+            );
+        }
+
+        scalarField cellWeights;
+        word weightName;
+        if (model.readIfPresent("weightField", weightName))
+        {
+            volScalarField weights
+            (
+                IOobject
+                (
+                    weightName,
+                    mesh.time().timeName(),
+                    mesh,
+                    IOobject::MUST_READ,
+                    IOobject::NO_WRITE
+                ),
+                mesh
+            );
+            cellWeights = weights.primitiveField();
+        }
+
+        decompositionMethod& method = model.decomposer();
+
+        CompactListList<label> cellCells;
+        decompositionMethod::calcCellCells
+        (
+            mesh,
+            identity(mesh.nCells()),
+            mesh.nCells(),
+            false,
+            cellCells
+        );
+
+        labelList cellToProc = method.decompose(mesh, cellWeights);
+
+        Info<< "\nFinished decomposition into "
+            << method.nDomains() << " domains in "
+            << decompositionTime.elapsedCpuTime()
+            << " s" << nl << endl;
+
+        decompositionInformation info
+        (
+            cellCells,
+            cellToProc,
+            method.nDomains()
+        );
+
+        if (verbose)
+        {
+            info.printDetails(Info);
+            Info<< nl;
+        }
+        info.printSummary(Info);
+    }
+
+    Info<< "\nEnd\n" << endl;
+
+    return 0;
+}
+
+
+// ************************************************************************* //

applications/test/foamToMetisGraph/Make/files

@@ -0,0 +1,3 @@
+foamToMetisGraph.C
+
+EXE = $(FOAM_USER_APPBIN)/foamToMetisGraph

applications/test/foamToMetisGraph/Make/options

@@ -0,0 +1,3 @@
+EXE_INC =
+
+EXE_LIBS =

applications/test/foamToMetisGraph/foamToMetisGraph.C

@@ -0,0 +1,86 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | Copyright (C) 2017 OpenCFD Ltd.
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+Description
+    Create a metis graph file representation of an OpenFOAM mesh
+
+\*---------------------------------------------------------------------------*/
+
+#include "argList.H"
+#include "Time.H"
+#include "polyMesh.H"
+#include "OFstream.H"
+
+using namespace Foam;
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+// Main program:
+
+int main(int argc, char *argv[])
+{
+    argList::noParallel();
+    argList::noFunctionObjects();
+    argList::addNote
+    (
+        "Create a metis graph file representation for an OpenFOAM mesh"
+    );
+
+    #include "setRootCase.H"
+    #include "createTime.H"
+    #include "createPolyMesh.H"
+
+    const labelListList& cellCells = mesh.cellCells();
+
+    // No. of Nodes = nCells
+    // No. of Edges connecting Nodes = nInternalFaces
+
+    OFstream os(args.caseName() + ".graph", IOstream::ASCII);
+
+    os  << "%% metis graph file, of an OpenFOAM mesh %%" << nl
+        << "%% nCells=" << mesh.nCells()
+        << " nFaces=" << mesh.nFaces()
+        << " nInternalFaces=" << mesh.nInternalFaces() << nl;
+
+    os  << cellCells.size() << " " << mesh.nInternalFaces() << nl;
+
+    for (const auto& edges : cellCells)
+    {
+        forAll(edges, i)
+        {
+            if (i) os << " ";
+            os << edges[i] + 1;  // index starts at 1.
+        }
+        os << nl;
+    }
+
+    Info<<"Wrote graph with "
+        << mesh.nCells() << " nodes and "
+        << mesh.nInternalFaces() << " edges to "
+        << os.name() << nl;
+
+    Info<< nl << "End\n" << endl;
+
+    return 0;
+}
+
+// ************************************************************************* //