diff --git a/src/parallel/decompose/Allwclean b/src/parallel/decompose/Allwclean index 402ddedc7a..bb8efaecaa 100755 --- a/src/parallel/decompose/Allwclean +++ b/src/parallel/decompose/Allwclean @@ -16,30 +16,39 @@ fi # Define how to clean an mpi-versioned library wcleanMpiLib() { - for libName - do ( + version="$1" + libName="$2" + WM_OPTIONS="$WM_OPTIONS$WM_MPLIB" whichmpi="$WM_PROJECT_DIR/platforms/$WM_OPTIONS/src/parallel/decompose/$libName/using:$FOAM_MPI" - whichscotch="$WM_PROJECT_DIR/platforms/$WM_OPTIONS/src/parallel/decompose/$libName/using:$SCOTCH_VERSION" + whichscotch="$WM_PROJECT_DIR/platforms/$WM_OPTIONS/src/parallel/decompose/$libName/using:$version" wclean $libName ) - done } + if [ -n "$SCOTCH_ARCH_PATH" ] then wclean scotchDecomp if [ -d "$FOAM_LIBBIN/$FOAM_MPI" ] then - wcleanMpiLib ptscotchDecomp + wcleanMpiLib $SCOTCH_VERSION ptscotchDecomp fi else echo " skipping scotchDecomp (ptscotchDecomp)" fi + +if [ -n "$ZOLTAN_ARCH_PATH" ] +then + wcleanMpiLib 3.90 zoltanDecomp +fi + + wclean metisDecomp wclean decompositionMethods + #------------------------------------------------------------------------------ diff --git a/src/parallel/decompose/Allwmake b/src/parallel/decompose/Allwmake index 0c213e59c7..287a0922fd 100755 --- a/src/parallel/decompose/Allwmake +++ b/src/parallel/decompose/Allwmake @@ -22,33 +22,52 @@ fi # use sentinel file to handle version changes wmakeMpiLib() { - for libName - do ( + version="$1" + libName="$2" + WM_OPTIONS="$WM_OPTIONS$WM_MPLIB" whichmpi="$WM_PROJECT_DIR/platforms/$WM_OPTIONS/src/parallel/decompose/$libName/using:$FOAM_MPI" - whichscotch="$WM_PROJECT_DIR/platforms/$WM_OPTIONS/src/parallel/decompose/$libName/using:$SCOTCH_VERSION" - [ -e "$whichmpi" -a -e "$whichscotch" ] || wclean $libName + whichtarget="$WM_PROJECT_DIR/platforms/$WM_OPTIONS/src/parallel/decompose/$libName/using:$version" + [ -e "$whichmpi" -a -e "$whichtarget" ] || wclean $libName echo "wmake $targetType $libName" wmake $targetType $libName - touch "$whichmpi" "$whichscotch" + touch "$whichmpi" "$whichtarget" ) - done } + if [ -n "$SCOTCH_ARCH_PATH" ] then wmake $targetType scotchDecomp if [ -n "$WM_MPLIB" ] && [ -d "$FOAM_LIBBIN/$FOAM_MPI" ] then - wmakeMpiLib ptscotchDecomp + wmakeMpiLib $SCOTCH_VERSION ptscotchDecomp fi else echo " skipping scotchDecomp (ptscotchDecomp)" fi +if settings=`$WM_PROJECT_DIR/bin/foamEtcFile config.sh/zoltan` +then + . $settings + echo " using ZOLTAN_ARCH_PATH=$ZOLTAN_ARCH_PATH" + + if [ -n "$ZOLTAN_ARCH_PATH" ] && [ -n "$WM_MPLIB" ] && [ -d "$FOAM_LIBBIN/$FOAM_MPI" ] + then + wmakeMpiLib $ZOLTAN_VERSION zoltanDecomp + else + echo " skipping zoltanDecomp" + fi +else + echo + echo " Error: no config.sh/zoltan settings" + echo +fi + + # Try and build metisDecomp (has own logic) (cd metisDecomp && ./Allwmake $targetType) diff --git a/src/parallel/decompose/decompositionMethods/decompositionMethod/decompositionMethod.C b/src/parallel/decompose/decompositionMethods/decompositionMethod/decompositionMethod.C index 4023e2a736..4ecbbb0761 100644 --- a/src/parallel/decompose/decompositionMethods/decompositionMethod/decompositionMethod.C +++ b/src/parallel/decompose/decompositionMethods/decompositionMethod/decompositionMethod.C @@ -108,6 +108,13 @@ Foam::decompositionMethod::NewDecomposer Info<< "Selecting decomposer " << methodType << endl; + libs.open + ( + decompositionDict, + "libs", + decomposerConstructorTablePtr_ + ); + decomposerConstructorTable::iterator cstrIter = decomposerConstructorTablePtr_->find(methodType); @@ -141,6 +148,13 @@ Foam::decompositionMethod::NewDistributor Info<< "Selecting distributor " << methodType << endl; + libs.open + ( + distributionDict, + "libs", + distributorConstructorTablePtr_ + ); + distributorConstructorTable::iterator cstrIter = distributorConstructorTablePtr_->find(methodType); @@ -288,7 +302,7 @@ void Foam::decompositionMethod::calcCellCells // Get agglomerate owner on other side of coupled faces // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - labelList globalNeighbour(mesh.nFaces()-mesh.nInternalFaces()); + labelList globalNeighbour(mesh.nFaces() - mesh.nInternalFaces()); forAll(patches, patchi) { @@ -338,7 +352,7 @@ void Foam::decompositionMethod::calcCellCells if (pp.coupled() && (parallel || !isA(pp))) { label facei = pp.start(); - label bFacei = pp.start()-mesh.nInternalFaces(); + label bFacei = pp.start() - mesh.nInternalFaces(); forAll(pp, i) { @@ -389,7 +403,7 @@ void Foam::decompositionMethod::calcCellCells if (pp.coupled() && (parallel || !isA(pp))) { label facei = pp.start(); - label bFacei = pp.start()-mesh.nInternalFaces(); + label bFacei = pp.start() - mesh.nInternalFaces(); forAll(pp, i) { @@ -479,7 +493,7 @@ void Foam::decompositionMethod::calcCellCells // Get agglomerate owner on other side of coupled faces // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - labelList globalNeighbour(mesh.nFaces()-mesh.nInternalFaces()); + labelList globalNeighbour(mesh.nFaces() - mesh.nInternalFaces()); forAll(patches, patchi) { diff --git a/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C b/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C index 53f1e9cd6f..e4c0c80cf8 100644 --- a/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C +++ b/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C @@ -796,6 +796,7 @@ Foam::labelList Foam::ptscotchDecomp::decompose { decomp[i] = finalDecomp[i]; } + return decomp; } diff --git a/src/parallel/decompose/zoltanDecomp/Make/files b/src/parallel/decompose/zoltanDecomp/Make/files new file mode 100644 index 0000000000..761e86cd80 --- /dev/null +++ b/src/parallel/decompose/zoltanDecomp/Make/files @@ -0,0 +1,3 @@ +zoltanDecomp.C + +LIB = $(FOAM_LIBBIN)/$(FOAM_MPI)/libzoltanDecomp diff --git a/src/parallel/decompose/zoltanDecomp/Make/options b/src/parallel/decompose/zoltanDecomp/Make/options new file mode 100644 index 0000000000..a9e798f03d --- /dev/null +++ b/src/parallel/decompose/zoltanDecomp/Make/options @@ -0,0 +1,10 @@ +-include $(GENERAL_RULES)/mplibType + +EXE_INC = \ + $(PFLAGS) $(PINC) \ + -I$(FOAM_SRC)/Pstream/mpi/lnInclude \ + -I$(ZOLTAN_ARCH_PATH)/include/ \ + -I../decompositionMethods/lnInclude + +LIB_LIBS = \ + -L$(FOAM_EXT_LIBBIN)/$(FOAM_MPI) -lzoltan diff --git a/src/parallel/decompose/zoltanDecomp/zoltanDecomp.C b/src/parallel/decompose/zoltanDecomp/zoltanDecomp.C new file mode 100644 index 0000000000..aa8bda2b64 --- /dev/null +++ b/src/parallel/decompose/zoltanDecomp/zoltanDecomp.C @@ -0,0 +1,537 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | OpenFOAM: The Open Source CFD Toolbox + \\ / O peration | Website: https://openfoam.org + \\ / A nd | Copyright (C) 2021 OpenFOAM Foundation + \\/ M anipulation | +------------------------------------------------------------------------------- +License + This file is part of OpenFOAM. + + OpenFOAM is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with OpenFOAM. If not, see . + +\*---------------------------------------------------------------------------*/ + +#include "zoltanDecomp.H" +#include "globalIndex.H" +#include "PstreamGlobals.H" +#include "addToRunTimeSelectionTable.H" + +#include "sigFpe.H" +#ifdef LINUX_GNUC + #ifndef __USE_GNU + #define __USE_GNU + #endif + #include +#endif + +#include "zoltan.h" + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +namespace Foam +{ + defineTypeNameAndDebug(zoltanDecomp, 0); + + addToRunTimeSelectionTable + ( + decompositionMethod, + zoltanDecomp, + distributor + ); +} + + +// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * // + +static int get_number_of_vertices(void *data, int *ierr) +{ + const Foam::pointField& points = + *static_cast(data); + + *ierr = ZOLTAN_OK; + + return points.size(); +} + + +static void get_vertex_list +( + void* data, + int nGID, + int nLID, + ZOLTAN_ID_PTR globalIDs, + ZOLTAN_ID_PTR localIDs, + int wgt_dim, + float* obj_wgts, + int* ierr +) +{ + const Foam::Tuple2& + vertexData = + *static_cast + < + const Foam::Tuple2 + < + const Foam::pointField&, + const Foam::scalarField& + >* + >(data); + + const Foam::pointField& points = vertexData.first(); + const Foam::scalarField& weights = vertexData.second(); + + Foam::globalIndex globalMap(points.size()); + + for (Foam::label i=0; i(data); + + if + ( + (nGID != 1) + || (nLID != 1) + || (nPoints != points.size()) + || (nDim != 3) + ) + { + *ierr = ZOLTAN_FATAL; + return; + } + + double* p = vertices; + + for (Foam::label celli=0; celli& adjacency = + *static_cast*>(data); + + Foam::labelList sizes(adjacency.sizes()); + + if ((nGID != 1) || (nLID != 1) || (nPoints != sizes.size())) + { + *ierr = ZOLTAN_FATAL; + return; + } + + for (Foam::label i=0; i& adjacency = + *static_cast*>(data); + + const Foam::labelList& m(adjacency.m()); + + Foam::globalIndex globalMap(nPoints); + + if + ( + (nGID != 1) + || (nLID != 1) + || (wgt_dim != 0) + ) + { + *ierr = ZOLTAN_FATAL; + return; + } + + forAll(m, i) + { + nborGID[i] = m[i]; + nborProc[i] = globalMap.whichProcID(m[i]); + } + + *ierr = ZOLTAN_OK; +} + + +Foam::label Foam::zoltanDecomp::decompose +( + const CompactListList