diff --git a/applications/utilities/mesh/manipulation/renumberMesh/Allwmake b/applications/utilities/mesh/manipulation/renumberMesh/Allwmake index ee135cc373..8fbaba1674 100755 --- a/applications/utilities/mesh/manipulation/renumberMesh/Allwmake +++ b/applications/utilities/mesh/manipulation/renumberMesh/Allwmake @@ -13,11 +13,11 @@ then export LINK_FLAGS="${LINK_FLAGS} -lSloanRenumber" fi -if [ -f "${ZOLTAN_ARCH_PATH}/lib/libzoltan.a" -a -f "${FOAM_LIBBIN}/libzoltanRenumber.so" ] +if [ -f "${ZOLTAN_ARCH_PATH}/lib64/libzoltan.a" -a -f "${FOAM_LIBBIN}/libzoltanRenumber.so" ] then echo "Found libzoltanRenumber.so -- enabling zoltan renumbering support." export COMPILE_FLAGS="-DFOAM_USE_ZOLTAN" - export LINK_FLAGS="${LINK_FLAGS} -lzoltanRenumber -L${ZOLTAN_ARCH_PATH}/lib -lzoltan" + export LINK_FLAGS="${LINK_FLAGS} -lzoltanRenumber -L${ZOLTAN_ARCH_PATH}/lib64 -lzoltan" fi wmake $targetType diff --git a/src/renumber/zoltanRenumber/zoltanRenumber.C b/src/renumber/zoltanRenumber/zoltanRenumber.C index 45aea6b156..ab3138d79a 100644 --- a/src/renumber/zoltanRenumber/zoltanRenumber.C +++ b/src/renumber/zoltanRenumber/zoltanRenumber.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | Website: https://openfoam.org - \\ / A nd | Copyright (C) 2011-2019 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2011-2021 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -21,31 +21,6 @@ License You should have received a copy of the GNU General Public License along with OpenFOAM. If not, see . -Class - Foam::zoltanRenumber - -Description - Renumber using Zoltan - - Zoltan install: - - in your ~/.bashrc: - export ZOLTAN_ARCH_DIR=\ - $WM_THIRD_PARTY_DIR/platforms/linux64Gcc/Zoltan_XXX - - unpack into $WM_THIRD_PARTY_DIR - - cd Zoltan_XXX - - mkdir build - - cd build - - export CCFLAGS="-fPIC" - - export CXXFLAGS="-fPIC" - - export CFLAGS="-fPIC" - - export LDFLAGS="-shared" - - ../configure \ - --prefix=$ZOLTAN_ARCH_DIR \ - --with-ccflags=-fPIC --with-cxxflags=-fPIC --with-ldflags=-shared - -SourceFiles - zoltanRenumber.C - \*---------------------------------------------------------------------------*/ #include "zoltanRenumber.H" @@ -59,7 +34,6 @@ SourceFiles #include "PstreamGlobals.H" #include "zoltan.h" -#include // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // @@ -79,58 +53,76 @@ namespace Foam static int get_number_of_vertices(void *data, int *ierr) { const Foam::polyMesh& mesh = *static_cast(data); + *ierr = ZOLTAN_OK; + return mesh.nCells(); } -static void get_vertex_list(void *data, int sizeGID, int sizeLID, - ZOLTAN_ID_PTR globalID, ZOLTAN_ID_PTR localID, - int wgt_dim, float *obj_wgts, int *ierr) +static void get_vertex_list +( + void* data, + int nGID, + int nLID, + ZOLTAN_ID_PTR globalIDs, + ZOLTAN_ID_PTR localIDs, + int wgt_dim, + float* obj_wgts, + int* ierr +) { const Foam::polyMesh& mesh = *static_cast(data); - *ierr = ZOLTAN_OK; + // In this example, return the IDs of our vertices, but no weights. + // Zoltan will assume equally weighted vertices. - /* In this example, return the IDs of our vertices, but no weights. - * Zoltan will assume equally weighted vertices. - */ - - wgt_dim = 0; - obj_wgts = nullptr; + // Global calculation engine + Foam::globalIndex globalCellMap(mesh.nCells()); for (Foam::label i=0; i(data); - if ((sizeGID != 1) || (sizeLID != 1) || (num_obj != mesh.nCells())) + if ((nGID != 1) || (nLID != 1) || (nCells != mesh.nCells())) { *ierr = ZOLTAN_FATAL; return; } - for (Foam::label i=0; i < num_obj ;i++) + for (Foam::label i=0; i(data); if ( - (sizeGID != 1) - || (sizeLID != 1) - || (num_obj != mesh.nCells()) + (nGID != 1) + || (nLID != 1) + || (nCells != mesh.nCells()) || (wgt_dim != 1) ) { @@ -163,19 +165,21 @@ static void get_edge_list(void *data, int sizeGID, int sizeLID, int* nextProc = nborProc; float* nextWgt = ewgts; - for (Foam::label i=0; i < num_obj; i++) + for (Foam::label i=0; i < nCells; i++) { - Foam::label celli = localID[i]; - + const Foam::label celli = localIDs[i]; const Foam::cell& cFaces = mesh.cells()[celli]; + forAll(cFaces, cFacei) { Foam::label n = 0; - Foam::label facei = cFaces[cFacei]; + const Foam::label facei = cFaces[cFacei]; + if (mesh.isInternalFace(facei)) { Foam::label nbr = mesh.faceOwner()[facei]; + if (nbr == celli) { nbr = mesh.faceNeighbour()[facei]; @@ -188,6 +192,7 @@ static void get_edge_list(void *data, int sizeGID, int sizeLID, n++; } + if (n != num_edges[i]) { *ierr = ZOLTAN_FATAL; @@ -195,11 +200,12 @@ static void get_edge_list(void *data, int sizeGID, int sizeLID, } } } + *ierr = ZOLTAN_OK; } -static int get_mesh_dim(void *data, int *ierr) +static int get_mesh_dim(void* data, int* ierr) { const Foam::polyMesh& mesh = *static_cast(data); @@ -209,43 +215,41 @@ static int get_mesh_dim(void *data, int *ierr) static void get_geom_list ( - void *data, - int num_gid_entries, - int num_lid_entries, - int num_obj, - ZOLTAN_ID_PTR global_ids, - ZOLTAN_ID_PTR local_ids, - int num_dim, - double *geom_vec, - int *ierr + void* data, + int nGID, + int nLID, + int nCells, + ZOLTAN_ID_PTR globalIDs, + ZOLTAN_ID_PTR localIDs, + int nDim, + double* cellCentres, + int* ierr ) { const Foam::polyMesh& mesh = *static_cast(data); if ( - (num_gid_entries != 1) - || (num_lid_entries != 1) - || (num_obj != mesh.nCells()) - || (num_dim != mesh.nSolutionD()) + (nGID != 1) + || (nLID != 1) + || (nCells != mesh.nCells()) + || (nDim != mesh.nSolutionD()) ) { *ierr = ZOLTAN_FATAL; return; } - double* p = geom_vec; - + double* p = cellCentres; const Foam::Vector& sol = mesh.solutionD(); - const Foam::pointField& cc = mesh.cellCentres(); - for (Foam::label celli = 0; celli < num_obj; celli++) + for (Foam::label celli=0; celli(pMesh); - - - forAllConstIter(IDLList, coeffsDict_, iter) { - if (!iter().isDict()) + forAllConstIter(IDLList, coeffsDict_, iter) { - const word& key = iter().keyword(); - const word value(iter().stream()); + if (!iter().isDict()) + { + const word& key = iter().keyword(); + const word value(iter().stream()); - Info<< typeName << " : setting parameter " << key - << " to " << value << endl; + Info<< typeName << " : setting parameter " << key + << " to " << value << endl; - Zoltan_Set_Param(zz, key.c_str(), value.c_str()); + Zoltan_Set_Param(zz, key.c_str(), value.c_str()); + } } + + void* meshPtr = &const_cast(mesh); + + Zoltan_Set_Param(zz, "ORDER_TYPE", "LOCAL"); + + Zoltan_Set_Num_Obj_Fn(zz, get_number_of_vertices, meshPtr); + Zoltan_Set_Obj_List_Fn(zz, get_vertex_list, meshPtr); + + // Callbacks for geometry + Zoltan_Set_Num_Geom_Fn(zz, get_mesh_dim, meshPtr); + Zoltan_Set_Geom_Multi_Fn(zz, get_geom_list, meshPtr); + + // Callbacks for connectivity + Zoltan_Set_Num_Edges_Multi_Fn(zz, get_num_edges_list, meshPtr); + Zoltan_Set_Edge_List_Multi_Fn(zz, get_edge_list, meshPtr); } + // Local to global cell index mapper + globalIndex globalCellMap(mesh.nCells()); - Zoltan_Set_Num_Obj_Fn(zz, get_number_of_vertices, &mesh); - Zoltan_Set_Obj_List_Fn(zz, get_vertex_list, &mesh); - - // Callbacks for geometry - Zoltan_Set_Num_Geom_Fn(zz, get_mesh_dim, &mesh); - Zoltan_Set_Geom_Multi_Fn(zz, get_geom_list, &mesh); - - // Callbacks for connectivity - Zoltan_Set_Num_Edges_Multi_Fn(zz, get_num_edges_list, &mesh); - Zoltan_Set_Edge_List_Multi_Fn(zz, get_edge_list, &mesh); - - - - // Note: !global indices - List wantedCells(mesh.nCells()); - - globalIndex globalCells(mesh.nCells()); - forAll(wantedCells, i) + List globalCells(mesh.nCells()); + forAll(globalCells, i) { - // wantedCells[i] = i; - wantedCells[i] = globalCells.toGlobal(i); + globalCells[i] = globalCellMap.toGlobal(i); } + // Old local to new global cell index map + // Values set by Zoltan_Order List oldToNew(mesh.nCells()); + // Call Zoltan to reorder the cells int err = Zoltan_Order ( zz, - 1, // int num_gid_entries, - mesh.globalData().nTotalCells(), // int num_obj, - wantedCells.begin(), + 1, // int nGID, + mesh.nCells(), + globalCells.begin(), oldToNew.begin() ); + // Check for Zoltan errors if (err != ZOLTAN_OK) { FatalErrorInFunction << "Failed Zoltan_Order" << exit(FatalError); } - + // Free the argv array for (label i = 0; i < argc; i++) { free(argv[i]); } + // Free the Zoltan_Struct allocated by Zoltan_Create + Zoltan_Destroy(&zz); + // Map the oldToNew global cell indices to local labelList order(oldToNew.size()); forAll(order, i) { - order[i] = oldToNew[i]; + order[i] = globalCellMap.toLocal(oldToNew[i]); } + return order; } diff --git a/src/renumber/zoltanRenumber/zoltanRenumber.H b/src/renumber/zoltanRenumber/zoltanRenumber.H index 533813a01b..3f23995a0b 100644 --- a/src/renumber/zoltanRenumber/zoltanRenumber.H +++ b/src/renumber/zoltanRenumber/zoltanRenumber.H @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | Website: https://openfoam.org - \\ / A nd | Copyright (C) 2011-2020 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2011-2021 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -25,7 +25,26 @@ Class Foam::zoltanRenumber Description - Use Zoltan + Use Zoltan for renumbering a case in parallel. + + Note: Zoltan methods do not support serial operation. + + Zoltan install: + - in your ~/.bashrc: + export ZOLTAN_ARCH_PATH=\ + $WM_THIRD_PARTY_DIR/platforms/linux64Gcc/Zoltan_XXX + - unpack into $WM_THIRD_PARTY_DIR + - cd Zoltan_XXX + - mkdir build + - cd build + - export CCFLAGS="-fPIC" + - export CXXFLAGS="-fPIC" + - export CFLAGS="-fPIC" + - ../configure \ + --prefix=$ZOLTAN_ARCH_DIR \ + --with-ccflags=-fPIC --with-cxxflags=-fPIC + - make everything + - make install SourceFiles zoltanRenumber.C