zhyang-dev/OpenFOAM-12
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

zoltanRenumber: Updated, corrected and tested

Browse Source
This commit is contained in:
Henry Weller
2021-12-15 11:45:11 +00:00
parent d9be4381f5
commit d90f421841
3 changed files with 149 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
applications/utilities/mesh/manipulation/renumberMesh/Allwmake
View File

@ -13,11 +13,11 @@ then
export LINK_FLAGS="${LINK_FLAGS} -lSloanRenumber"
fi
if [ -f "${ZOLTAN_ARCH_PATH}/lib/libzoltan.a" -a -f "${FOAM_LIBBIN}/libzoltanRenumber.so" ]
if [ -f "${ZOLTAN_ARCH_PATH}/lib64/libzoltan.a" -a -f "${FOAM_LIBBIN}/libzoltanRenumber.so" ]
then
echo "Found libzoltanRenumber.so -- enabling zoltan renumbering support."
export COMPILE_FLAGS="-DFOAM_USE_ZOLTAN"
export LINK_FLAGS="${LINK_FLAGS} -lzoltanRenumber -L${ZOLTAN_ARCH_PATH}/lib -lzoltan"
export LINK_FLAGS="${LINK_FLAGS} -lzoltanRenumber -L${ZOLTAN_ARCH_PATH}/lib64 -lzoltan"
fi
wmake $targetType

242
src/renumber/zoltanRenumber/zoltanRenumber.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2011-2019 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2021 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -21,31 +21,6 @@ License
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
Class
Foam::zoltanRenumber
Description
Renumber using Zoltan
Zoltan install:
- in your ~/.bashrc:
export ZOLTAN_ARCH_DIR=\
$WM_THIRD_PARTY_DIR/platforms/linux64Gcc/Zoltan_XXX
- unpack into $WM_THIRD_PARTY_DIR
- cd Zoltan_XXX
- mkdir build
- cd build
- export CCFLAGS="-fPIC"
- export CXXFLAGS="-fPIC"
- export CFLAGS="-fPIC"
- export LDFLAGS="-shared"
- ../configure \
--prefix=$ZOLTAN_ARCH_DIR \
--with-ccflags=-fPIC --with-cxxflags=-fPIC --with-ldflags=-shared
SourceFiles
zoltanRenumber.C
\*---------------------------------------------------------------------------*/
#include "zoltanRenumber.H"
@ -59,7 +34,6 @@ SourceFiles
#include "PstreamGlobals.H"
#include "zoltan.h"
#include <mpi.h>
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
@ -79,58 +53,76 @@ namespace Foam
static int get_number_of_vertices(void *data, int *ierr)
{
const Foam::polyMesh& mesh = *static_cast<const Foam::polyMesh*>(data);
*ierr = ZOLTAN_OK;
return mesh.nCells();
}
static void get_vertex_list(void *data, int sizeGID, int sizeLID,
ZOLTAN_ID_PTR globalID, ZOLTAN_ID_PTR localID,
int wgt_dim, float *obj_wgts, int *ierr)
static void get_vertex_list
(
void* data,
int nGID,
int nLID,
ZOLTAN_ID_PTR globalIDs,
ZOLTAN_ID_PTR localIDs,
int wgt_dim,
float* obj_wgts,
int* ierr
)
{
const Foam::polyMesh& mesh = *static_cast<const Foam::polyMesh*>(data);
*ierr = ZOLTAN_OK;
// In this example, return the IDs of our vertices, but no weights.
// Zoltan will assume equally weighted vertices.
/* In this example, return the IDs of our vertices, but no weights.
* Zoltan will assume equally weighted vertices.
*/
wgt_dim = 0;
obj_wgts = nullptr;
// Global calculation engine
Foam::globalIndex globalCellMap(mesh.nCells());
for (Foam::label i=0; i<mesh.nCells(); i++)
{
globalID[i] = i; // should be global
localID[i] = i;
localIDs[i] = i;
globalIDs[i] = globalCellMap.toGlobal(i);
}
*ierr = ZOLTAN_OK;
}
static void get_num_edges_list(void *data, int sizeGID, int sizeLID,
int num_obj,
ZOLTAN_ID_PTR globalID, ZOLTAN_ID_PTR localID,
int *numEdges, int *ierr)
static void get_num_edges_list
(
void* data,
int nGID,
int nLID,
int nCells,
ZOLTAN_ID_PTR globalIDs,
ZOLTAN_ID_PTR localIDs,
int* numEdges,
int* ierr
)
{
const Foam::polyMesh& mesh = *static_cast<const Foam::polyMesh*>(data);
if ((sizeGID != 1) || (sizeLID != 1) || (num_obj != mesh.nCells()))
if ((nGID != 1) || (nLID != 1) || (nCells != mesh.nCells()))
{
*ierr = ZOLTAN_FATAL;
return;
}
for (Foam::label i=0; i < num_obj ;i++)
for (Foam::label i=0; i<nCells; i++)
{
Foam::label celli = localID[i];
const Foam::cell& cFaces = mesh.cells()[celli];
const Foam::cell& cFaces = mesh.cells()[localIDs[i]];
forAll(cFaces, cFacei)
{
Foam::label n = 0;
if (mesh.isInternalFace(cFaces[cFacei]))
{
n++;
}
numEdges[i] = n;
}
}
@ -139,19 +131,29 @@ static void get_num_edges_list(void *data, int sizeGID, int sizeLID,
}
static void get_edge_list(void *data, int sizeGID, int sizeLID,
int num_obj, ZOLTAN_ID_PTR globalID, ZOLTAN_ID_PTR localID,
int *num_edges,
ZOLTAN_ID_PTR nborGID, int *nborProc,
int wgt_dim, float *ewgts, int *ierr)
static void get_edge_list
(
void* data,
int nGID,
int nLID,
int nCells,
ZOLTAN_ID_PTR globalIDs,
ZOLTAN_ID_PTR localIDs,
int* num_edges,
ZOLTAN_ID_PTR nborGID,
int* nborProc,
int wgt_dim,
float* ewgts,
int* ierr
)
{
const Foam::polyMesh& mesh = *static_cast<const Foam::polyMesh*>(data);
if
(
(sizeGID != 1)
|| (sizeLID != 1)
|| (num_obj != mesh.nCells())
(nGID != 1)
|| (nLID != 1)
|| (nCells != mesh.nCells())
|| (wgt_dim != 1)
)
{
@ -163,19 +165,21 @@ static void get_edge_list(void *data, int sizeGID, int sizeLID,
int* nextProc = nborProc;
float* nextWgt = ewgts;
for (Foam::label i=0; i < num_obj; i++)
for (Foam::label i=0; i < nCells; i++)
{
Foam::label celli = localID[i];
const Foam::label celli = localIDs[i];
const Foam::cell& cFaces = mesh.cells()[celli];
forAll(cFaces, cFacei)
{
Foam::label n = 0;
Foam::label facei = cFaces[cFacei];
const Foam::label facei = cFaces[cFacei];
if (mesh.isInternalFace(facei))
{
Foam::label nbr = mesh.faceOwner()[facei];
if (nbr == celli)
{
nbr = mesh.faceNeighbour()[facei];
@ -188,6 +192,7 @@ static void get_edge_list(void *data, int sizeGID, int sizeLID,
n++;
}
if (n != num_edges[i])
{
*ierr = ZOLTAN_FATAL;
@ -195,11 +200,12 @@ static void get_edge_list(void *data, int sizeGID, int sizeLID,
}
}
}
*ierr = ZOLTAN_OK;
}
static int get_mesh_dim(void *data, int *ierr)
static int get_mesh_dim(void* data, int* ierr)
{
const Foam::polyMesh& mesh = *static_cast<const Foam::polyMesh*>(data);
@ -209,43 +215,41 @@ static int get_mesh_dim(void *data, int *ierr)
static void get_geom_list
(
void *data,
int num_gid_entries,
int num_lid_entries,
int num_obj,
ZOLTAN_ID_PTR global_ids,
ZOLTAN_ID_PTR local_ids,
int num_dim,
double *geom_vec,
int *ierr
void* data,
int nGID,
int nLID,
int nCells,
ZOLTAN_ID_PTR globalIDs,
ZOLTAN_ID_PTR localIDs,
int nDim,
double* cellCentres,
int* ierr
)
{
const Foam::polyMesh& mesh = *static_cast<const Foam::polyMesh*>(data);
if
(
(num_gid_entries != 1)
|| (num_lid_entries != 1)
|| (num_obj != mesh.nCells())
|| (num_dim != mesh.nSolutionD())
(nGID != 1)
|| (nLID != 1)
|| (nCells != mesh.nCells())
|| (nDim != mesh.nSolutionD())
)
{
*ierr = ZOLTAN_FATAL;
return;
}
double* p = geom_vec;
double* p = cellCentres;
const Foam::Vector<Foam::label>& sol = mesh.solutionD();
const Foam::pointField& cc = mesh.cellCentres();
for (Foam::label celli = 0; celli < num_obj; celli++)
for (Foam::label celli=0; celli<nCells; celli++)
{
const Foam::point& pt = cc[celli];
for (Foam::direction cmpt = 0; cmpt < Foam::vector::nComponents; cmpt++)
for (Foam::direction cmpt=0; cmpt<Foam::vector::nComponents; cmpt++)
{
if (sol[cmpt] == 1)
{
@ -253,6 +257,7 @@ static void get_geom_list
}
}
}
*ierr = ZOLTAN_OK;
}
@ -270,7 +275,7 @@ Foam::zoltanRenumber::zoltanRenumber(const dictionary& renumberDict)
Foam::labelList Foam::zoltanRenumber::renumber
(
const polyMesh& pMesh,
const polyMesh& mesh,
const pointField& points
) const
{
@ -287,8 +292,6 @@ Foam::labelList Foam::zoltanRenumber::renumber
float ver;
int rc = Zoltan_Initialize(argc, argv, &ver);
Foam::Pout<< "Initialised to " << ver << Foam::endl;
if (rc != ZOLTAN_OK)
{
FatalErrorInFunction
@ -297,76 +300,83 @@ Foam::labelList Foam::zoltanRenumber::renumber
struct Zoltan_Struct *zz = Zoltan_Create(PstreamGlobals::MPI_COMM_FOAM);
polyMesh& mesh = const_cast<polyMesh&>(pMesh);
forAllConstIter(IDLList<entry>, coeffsDict_, iter)
{
if (!iter().isDict())
forAllConstIter(IDLList<entry>, coeffsDict_, iter)
{
const word& key = iter().keyword();
const word value(iter().stream());
if (!iter().isDict())
{
const word& key = iter().keyword();
const word value(iter().stream());
Info<< typeName << " : setting parameter " << key
<< " to " << value << endl;
Info<< typeName << " : setting parameter " << key
<< " to " << value << endl;
Zoltan_Set_Param(zz, key.c_str(), value.c_str());
Zoltan_Set_Param(zz, key.c_str(), value.c_str());
}
}
void* meshPtr = &const_cast<polyMesh&>(mesh);
Zoltan_Set_Param(zz, "ORDER_TYPE", "LOCAL");
Zoltan_Set_Num_Obj_Fn(zz, get_number_of_vertices, meshPtr);
Zoltan_Set_Obj_List_Fn(zz, get_vertex_list, meshPtr);
// Callbacks for geometry
Zoltan_Set_Num_Geom_Fn(zz, get_mesh_dim, meshPtr);
Zoltan_Set_Geom_Multi_Fn(zz, get_geom_list, meshPtr);
// Callbacks for connectivity
Zoltan_Set_Num_Edges_Multi_Fn(zz, get_num_edges_list, meshPtr);
Zoltan_Set_Edge_List_Multi_Fn(zz, get_edge_list, meshPtr);
}
// Local to global cell index mapper
globalIndex globalCellMap(mesh.nCells());
Zoltan_Set_Num_Obj_Fn(zz, get_number_of_vertices, &mesh);
Zoltan_Set_Obj_List_Fn(zz, get_vertex_list, &mesh);
// Callbacks for geometry
Zoltan_Set_Num_Geom_Fn(zz, get_mesh_dim, &mesh);
Zoltan_Set_Geom_Multi_Fn(zz, get_geom_list, &mesh);
// Callbacks for connectivity
Zoltan_Set_Num_Edges_Multi_Fn(zz, get_num_edges_list, &mesh);
Zoltan_Set_Edge_List_Multi_Fn(zz, get_edge_list, &mesh);
// Note: !global indices
List<ZOLTAN_ID_TYPE> wantedCells(mesh.nCells());
globalIndex globalCells(mesh.nCells());
forAll(wantedCells, i)
List<ZOLTAN_ID_TYPE> globalCells(mesh.nCells());
forAll(globalCells, i)
{
// wantedCells[i] = i;
wantedCells[i] = globalCells.toGlobal(i);
globalCells[i] = globalCellMap.toGlobal(i);
}
// Old local to new global cell index map
// Values set by Zoltan_Order
List<ZOLTAN_ID_TYPE> oldToNew(mesh.nCells());
// Call Zoltan to reorder the cells
int err = Zoltan_Order
(
zz,
1, // int num_gid_entries,
mesh.globalData().nTotalCells(), // int num_obj,
wantedCells.begin(),
1, // int nGID,
mesh.nCells(),
globalCells.begin(),
oldToNew.begin()
);
// Check for Zoltan errors
if (err != ZOLTAN_OK)
{
FatalErrorInFunction
<< "Failed Zoltan_Order" << exit(FatalError);
}
// Free the argv array
for (label i = 0; i < argc; i++)
{
free(argv[i]);
}
// Free the Zoltan_Struct allocated by Zoltan_Create
Zoltan_Destroy(&zz);
// Map the oldToNew global cell indices to local
labelList order(oldToNew.size());
forAll(order, i)
{
order[i] = oldToNew[i];
order[i] = globalCellMap.toLocal(oldToNew[i]);
}
return order;
}

23
src/renumber/zoltanRenumber/zoltanRenumber.H
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2011-2020 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2021 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -25,7 +25,26 @@ Class
Foam::zoltanRenumber
Description
Use Zoltan
Use Zoltan for renumbering a case in parallel.
Note: Zoltan methods do not support serial operation.
Zoltan install:
- in your ~/.bashrc:
export ZOLTAN_ARCH_PATH=\
$WM_THIRD_PARTY_DIR/platforms/linux64Gcc/Zoltan_XXX
- unpack into $WM_THIRD_PARTY_DIR
- cd Zoltan_XXX
- mkdir build
- cd build
- export CCFLAGS="-fPIC"
- export CXXFLAGS="-fPIC"
- export CFLAGS="-fPIC"
- ../configure \
--prefix=$ZOLTAN_ARCH_DIR \
--with-ccflags=-fPIC --with-cxxflags=-fPIC
- make everything
- make install
SourceFiles
zoltanRenumber.C