diff --git a/etc/config.sh/parMetis b/etc/config.sh/parMetis new file mode 100644 index 0000000000..000835f947 --- /dev/null +++ b/etc/config.sh/parMetis @@ -0,0 +1,40 @@ +#----------------------------------*-sh-*-------------------------------------- +# ========= | +# \\ / F ield | OpenFOAM: The Open Source CFD Toolbox +# \\ / O peration | Website: https://openfoam.org +# \\ / A nd | Copyright (C) 2024 OpenFOAM Foundation +# \\/ M anipulation | +#------------------------------------------------------------------------------ +# License +# This file is part of OpenFOAM. +# +# OpenFOAM is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# OpenFOAM is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License +# along with OpenFOAM. If not, see . +# +# File +# etc/config.sh/parMetis +# +# Description +# Setup file for parMetis include/libraries. +# Sourced during wmake process only. +# +# Note +# A csh version is not needed, since the values here are only sourced +# during the wmake process +# +#------------------------------------------------------------------------------ + +export PARMETIS_VERSION=parmetis-4.0.3 +export PARMETIS_ARCH_PATH=$WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER$WM_PRECISION_OPTION$WM_LABEL_OPTION/$PARMETIS_VERSION + +#------------------------------------------------------------------------------ diff --git a/src/parallel/decompose/parMetis/Allwclean b/src/parallel/decompose/parMetis/Allwclean new file mode 100755 index 0000000000..9907fc0c20 --- /dev/null +++ b/src/parallel/decompose/parMetis/Allwclean @@ -0,0 +1,18 @@ +#!/bin/sh +cd ${0%/*} || exit 1 # Run from this directory + +. $WM_PROJECT_DIR/wmake/scripts/AllwmakeMpiLib + +# Get PARMETIS_VERSION, PARMETIS_ARCH_PATH +if settings=`$WM_PROJECT_DIR/bin/foamEtcFile config.sh/parMetis` +then + . $settings + echo " using PARMETIS_ARCH_PATH=$PARMETIS_ARCH_PATH" + wcleanMpiLib $PARMETIS_VERSION parMetisDecomp +else + echo + echo " Error: no config.sh/parMetis settings" + echo +fi + +#------------------------------------------------------------------------------ diff --git a/src/parallel/decompose/parMetis/Allwmake b/src/parallel/decompose/parMetis/Allwmake new file mode 100755 index 0000000000..f91c8f3fc0 --- /dev/null +++ b/src/parallel/decompose/parMetis/Allwmake @@ -0,0 +1,24 @@ +#!/bin/sh +cd ${0%/*} || exit 1 # Run from this directory + +# Parse arguments for library compilation +. $WM_PROJECT_DIR/wmake/scripts/AllwmakeParseArguments + +. $WM_PROJECT_DIR/wmake/scripts/AllwmakeMpiLib + +# get PARMETIS_VERSION, PARMETIS_ARCH_PATH +if settings=`$WM_PROJECT_DIR/bin/foamEtcFile config.sh/parMetis` +then + . $settings + echo " using PARMETIS_ARCH_PATH=$PARMETIS_ARCH_PATH" + if [ -r $PARMETIS_ARCH_PATH/lib/libparmetis.so ] + then + wmakeMpiLib $PARMETIS_VERSION + fi +else + echo + echo " Error: no config.sh/parMetis settings" + echo +fi + +#------------------------------------------------------------------------------ diff --git a/src/parallel/decompose/parMetis/Make/files b/src/parallel/decompose/parMetis/Make/files new file mode 100644 index 0000000000..3d70ac1174 --- /dev/null +++ b/src/parallel/decompose/parMetis/Make/files @@ -0,0 +1,3 @@ +parMetis.C + +LIB = $(FOAM_LIBBIN)/$(FOAM_MPI)/libparMetisDecomp diff --git a/src/parallel/decompose/parMetis/Make/options b/src/parallel/decompose/parMetis/Make/options new file mode 100644 index 0000000000..fd73f69fc1 --- /dev/null +++ b/src/parallel/decompose/parMetis/Make/options @@ -0,0 +1,10 @@ +-include $(GENERAL_RULES)/mplibType + +EXE_INC = \ + $(PFLAGS) $(PINC) \ + -I$(FOAM_SRC)/Pstream/mpi/lnInclude \ + -I$(PARMETIS_ARCH_PATH)/include \ + -I../decompositionMethods/lnInclude + +LIB_LIBS = \ + -L$(PARMETIS_ARCH_PATH)/lib -lparmetis diff --git a/src/parallel/decompose/parMetis/parMetis.C b/src/parallel/decompose/parMetis/parMetis.C new file mode 100644 index 0000000000..ea56e82075 --- /dev/null +++ b/src/parallel/decompose/parMetis/parMetis.C @@ -0,0 +1,433 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | OpenFOAM: The Open Source CFD Toolbox + \\ / O peration | Website: https://openfoam.org + \\ / A nd | Copyright (C) 2024 OpenFOAM Foundation + \\/ M anipulation | +------------------------------------------------------------------------------- +License + This file is part of OpenFOAM. + + OpenFOAM is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with OpenFOAM. If not, see . + +\*---------------------------------------------------------------------------*/ + +#include "parMetis.H" +#include "Time.H" +#include "globalIndex.H" +#include "labelIOField.H" +#include "addToRunTimeSelectionTable.H" + +extern "C" +{ + #include "parmetis.h" +} + + +// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * // + +namespace Foam +{ +namespace decompositionMethods +{ + defineTypeNameAndDebug(parMetis, 0); + + addToRunTimeSelectionTable + ( + decompositionMethod, + parMetis, + distributor + ); +} +} + + +// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * // + +Foam::label Foam::decompositionMethods::parMetis::decompose +( + const labelList& xadj, + const labelList& adjncy, + const pointField& cellCentres, + const labelList& cellWeights, + const labelList& faceWeights, + labelList& decomp +) +{ + // C style numbering + label numFlag = 0; + + // Number of weights or balance constraints + label nWeights = cellWeights.size()/cellCentres.size(); + + scalarList processorWeights; + + if (processorWeights_.size()) + { + if (processorWeights_.size() != nWeights*nProcessors_) + { + FatalErrorInFunction + << "Number of processor weights specified in parMetisCoeffs " + << processorWeights_.size() + << " does not equal number of constraints * number of domains " + << nWeights*nProcessors_ + << exit(FatalError); + } + + processorWeights = processorWeights_; + } + else + { + processorWeights.setSize(nWeights*nProcessors_, 1.0/nProcessors_); + } + + // Imbalance tolerance + Field ubvec(nWeights, 1.02); + + // If only one processor there is no imbalance + if (nProcessors_ == 1) + { + ubvec[0] = 1; + } + + // Distribute to all processors the number of cells on each processor + globalIndex globalMap(cellCentres.size()); + + // Get the processor-cell offset table + labelList cellOffsets(globalMap.offsets()); + + // Weight info + label wgtFlag = 0; + const label* vwgtPtr = nullptr; + const label* adjwgtPtr = nullptr; + + // Weights on vertices of graph (cells) + if (cellWeights.size()) + { + vwgtPtr = cellWeights.begin(); + wgtFlag += 2; + } + + // Weights on edges of graph (faces) + if (faceWeights.size()) + { + adjwgtPtr = faceWeights.begin(); + wgtFlag += 1; + } + + MPI_Comm comm = MPI_COMM_WORLD; + + // Output: cell -> processor addressing + decomp.setSize(cellCentres.size()); + + // Output: the number of edges that are cut by the partitioning + label edgeCut = 0; + + if (method_ == "kWay") + { + ParMETIS_V3_PartKway + ( + cellOffsets.begin(), + const_cast(xadj.begin()), + const_cast(adjncy.begin()), + const_cast(vwgtPtr), + const_cast(adjwgtPtr), + &wgtFlag, + &numFlag, + &nWeights, + &nProcessors_, + processorWeights.begin(), + ubvec.begin(), + const_cast(options_).begin(), + &edgeCut, + decomp.begin(), + &comm + ); + } + else if (method_ == "geomKway") + { + // Number of dimensions + label nDims = 3; + + // Convert pointField into float + Field xyz(nDims*cellCentres.size()); + label i = 0; + forAll(cellCentres, celli) + { + const point& cc = cellCentres[celli]; + xyz[i++] = cc.x(); + xyz[i++] = cc.y(); + xyz[i++] = cc.z(); + } + + ParMETIS_V3_PartGeomKway + ( + cellOffsets.begin(), + const_cast(xadj.begin()), + const_cast(adjncy.begin()), + const_cast(vwgtPtr), + const_cast(adjwgtPtr), + &wgtFlag, + &numFlag, + &nDims, + xyz.begin(), + &nWeights, + &nProcessors_, + processorWeights.begin(), + ubvec.begin(), + const_cast(options_).begin(), + &edgeCut, + decomp.begin(), + &comm + ); + } + else if (method_ == "adaptiveRepart") + { + // Size of the vertices with respect to redistribution cost + labelList vsize(cellCentres.size(), 1); + + ParMETIS_V3_AdaptiveRepart + ( + cellOffsets.begin(), + const_cast(xadj.begin()), + const_cast(adjncy.begin()), + const_cast(vwgtPtr), + const_cast(vsize.begin()), + const_cast(adjwgtPtr), + &wgtFlag, + &numFlag, + &nWeights, + &nProcessors_, + processorWeights.begin(), + ubvec.begin(), + &itr_, + const_cast(options_).begin(), + &edgeCut, + decomp.begin(), + &comm + ); + } + + return edgeCut; +} + + +// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * // + +Foam::decompositionMethods::parMetis::parMetis +( + const dictionary& decompositionDict +) +: + decompositionMethod(decompositionDict), + method_("geomKway"), + options_(4, 0), + itr_(1000) +{ + // Check for user supplied weights and decomp options + if (decompositionDict.found("parMetisCoeffs")) + { + const dictionary& parMetisCoeffs = + decompositionDict.subDict("parMetisCoeffs"); + + Info<< type() << ": reading coefficients:" << endl; + + if (parMetisCoeffs.readIfPresent("method", method_)) + { + if + ( + method_ != "kWay" + && method_ != "geomKWay" + && method_ != "adaptiveRepart" + ) + { + FatalIOErrorInFunction(parMetisCoeffs) + << "Method " << method_ + << " in parMetisCoeffs in dictionary : " + << decompositionDict.name() + << " should be kWay, geomKWay or adaptiveRepart" + << exit(FatalIOError); + } + + Info<< " method: " << method_ << endl; + } + + if + ( + method_ == "adaptiveRepart" + && parMetisCoeffs.readIfPresent("itr", itr_) + ) + { + Info<< " itr: " << itr_ << endl; + } + + if (parMetisCoeffs.readIfPresent("options", options_)) + { + if (options_.size() != 4) + { + FatalIOErrorInFunction(parMetisCoeffs) + << "Number of options in parMetisCoeffs dictionary : " + << decompositionDict.name() + << " should be 4, found " << options_ + << exit(FatalIOError); + } + + Info<< " options: " << options_ << endl; + } + + parMetisCoeffs.readIfPresent("processorWeights_", processorWeights_); + + Info << endl; + } +} + + +// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // + +Foam::labelList Foam::decompositionMethods::parMetis::decompose +( + const polyMesh& mesh, + const pointField& points, + const scalarField& pointWeights +) +{ + if (points.size() != mesh.nCells()) + { + FatalErrorInFunction + << "Can use this decomposition method only for the whole mesh" + << endl + << "and supply one coordinate (cellCentre) for every cell." << endl + << "The number of coordinates " << points.size() << endl + << "The number of cells in the mesh " << mesh.nCells() + << exit(FatalError); + } + + // Make Metis CSR (Compressed Storage Format) storage + // adjncy : contains neighbours (= edges in graph) + // xadj(celli) : start of information in adjncy for celli + CompactListList