From 78dbbdd7d301d2cedd426e1c54dd1e8ebd22e454 Mon Sep 17 00:00:00 2001 From: Henry Weller Date: Sat, 1 Jun 2024 17:27:27 +0100 Subject: [PATCH] decompositionMethods::scaleWeights: Filter out zero weights The nWeights argument is now reduced if any of the weights are zero for all points on all processors and removed. --- .../decompositionMethod/decompositionMethod.C | 40 ++++++++++++++- .../decompositionMethod/decompositionMethod.H | 3 +- src/parallel/decompose/metis/metis.C | 32 ++++++------ src/parallel/decompose/parMetis/parMetis.C | 35 ++++++++++--- src/parallel/decompose/ptscotch/ptscotch.C | 5 +- src/parallel/decompose/scotch/scotch.C | 7 +-- src/parallel/decompose/zoltan/zoltan.C | 50 ++++++++++++------- 7 files changed, 126 insertions(+), 46 deletions(-) diff --git a/src/parallel/decompose/decompositionMethods/decompositionMethod/decompositionMethod.C b/src/parallel/decompose/decompositionMethods/decompositionMethod/decompositionMethod.C index bb2593fb8f..0928991964 100644 --- a/src/parallel/decompose/decompositionMethods/decompositionMethod/decompositionMethod.C +++ b/src/parallel/decompose/decompositionMethods/decompositionMethod/decompositionMethod.C @@ -321,7 +321,7 @@ Foam::labelList Foam::decompositionMethod::decompose Foam::labelList Foam::decompositionMethod::scaleWeights ( const scalarField& weights, - const label nWeights, + label& nWeights, const bool distributed ) { @@ -345,6 +345,41 @@ Foam::labelList Foam::decompositionMethod::scaleWeights intWeights[i] = ceil(scale*weights[i]); } + /* + // Alternatively calculate a separate scaling factor + // for each weight, it is not clear from the parMETIS or Scotch manuals + // which method should be used. + // + // Calculate the scalar -> integer scaling factors + scalarList sumWeights(nWeights, 0.0); + + forAll(weights, i) + { + sumWeights[i % nWeights] += weights[i]; + } + + if (distributed) + { + reduce(sumWeights, ListOp>()); + } + + scalarList scale(nWeights, 0.0); + forAll(scale, i) + { + if (sumWeights[i] > small) + { + scale[i] = labelMax/(2*sumWeights[i]); + } + } + + // Convert weights to integer + intWeights.setSize(weights.size()); + forAll(intWeights, i) + { + intWeights[i] = ceil(scale[i % nWeights]*weights[i]); + } + */ + // Calculate the sum over all processors of each weight labelList sumIntWeights(nWeights, 0); forAll(weights, i) @@ -383,6 +418,9 @@ Foam::labelList Foam::decompositionMethod::scaleWeights // Resize the weights list intWeights.setSize(nNonZeroWeights*(intWeights.size()/nWeights)); + + // Reset the number of weight to the number of non-zero weights + nWeights = nNonZeroWeights; } } diff --git a/src/parallel/decompose/decompositionMethods/decompositionMethod/decompositionMethod.H b/src/parallel/decompose/decompositionMethods/decompositionMethod/decompositionMethod.H index ab30813f7f..ef0daf044f 100644 --- a/src/parallel/decompose/decompositionMethods/decompositionMethod/decompositionMethod.H +++ b/src/parallel/decompose/decompositionMethods/decompositionMethod/decompositionMethod.H @@ -236,10 +236,11 @@ public: //- Convert the given scalar weights to labels // in the range 0-labelMax/2 + // Removes 0 weights and updates nWeights accordingly static labelList scaleWeights ( const scalarField& weights, - const label nWeights, + label& nWeights, const bool distributed = true ); diff --git a/src/parallel/decompose/metis/metis.C b/src/parallel/decompose/metis/metis.C index fe0468f374..4abb8792f4 100644 --- a/src/parallel/decompose/metis/metis.C +++ b/src/parallel/decompose/metis/metis.C @@ -79,8 +79,10 @@ Foam::label Foam::decompositionMethods::metis::decompose << exit(FatalError); } + label nWeights = 1; + // Cell weights (so on the vertices of the dual) - labelList intWeights(scaleWeights(cellWeights, 1, false)); + labelList intWeights(scaleWeights(cellWeights, nWeights, false)); // Face weights (so on the edges of the dual) labelList faceWeights; @@ -149,16 +151,16 @@ Foam::label Foam::decompositionMethods::metis::decompose { METIS_PartGraphRecursive ( - &numCells, // num vertices in graph - &ncon, // num balancing constraints + &numCells, // num vertices in graph + &ncon, // num balancing constraints const_cast(xadj).begin(), // indexing into adjncy const_cast(adjncy).begin(), // neighbour info - intWeights.begin(),// vertexweights - nullptr, // vsize: total communication vol - faceWeights.begin(),// edgeweights - &nProcs, // nParts + intWeights.begin(), // vertexweights + nullptr, // vsize: total communication vol + faceWeights.begin(), // edgeweights + &nProcs, // nParts processorWeights.begin(), // tpwgts - nullptr, // ubvec: processor imbalance (default) + nullptr, // ubvec: processor imbalance (default) options.begin(), &edgeCut, decomp.begin() @@ -168,16 +170,16 @@ Foam::label Foam::decompositionMethods::metis::decompose { METIS_PartGraphKway ( - &numCells, // num vertices in graph - &ncon, // num balancing constraints + &numCells, // num vertices in graph + &ncon, // num balancing constraints const_cast(xadj).begin(), // indexing into adjncy const_cast(adjncy).begin(), // neighbour info - intWeights.begin(),// vertexweights - nullptr, // vsize: total communication vol - faceWeights.begin(),// edgeweights - &nProcs, // nParts + intWeights.begin(), // vertexweights + nullptr, // vsize: total communication vol + faceWeights.begin(), // edgeweights + &nProcs, // nParts processorWeights.begin(), // tpwgts - nullptr, // ubvec: processor imbalance (default) + nullptr, // ubvec: processor imbalance (default) options.begin(), &edgeCut, decomp.begin() diff --git a/src/parallel/decompose/parMetis/parMetis.C b/src/parallel/decompose/parMetis/parMetis.C index 37c1b045ff..edfac18782 100644 --- a/src/parallel/decompose/parMetis/parMetis.C +++ b/src/parallel/decompose/parMetis/parMetis.C @@ -212,6 +212,26 @@ Foam::label Foam::decompositionMethods::parMetis::decompose ); } + // Sum the number of cells allocated to each processor + labelList nProcCells(Pstream::nProcs(), 0); + + forAll(decomp, i) + { + nProcCells[decomp[i]]++; + } + + reduce(nProcCells, ListOp>()); + + // If there are no cells allocated to this processor keep the first one + // to ensure that all processors have at least one cell + if (nProcCells[Pstream::myProcNo()] == 0) + { + Pout<< " No cells allocated to this processor" + ", keeping first cell" + << endl; + decomp[0] = Pstream::myProcNo(); + } + return edgeCut; } @@ -327,7 +347,8 @@ Foam::labelList Foam::decompositionMethods::parMetis::decompose cellCells ); - const label nWeights = this->nWeights(points, pointWeights); + label nWeights = this->nWeights(points, pointWeights); + const labelList intWeights(scaleWeights(pointWeights, nWeights)); labelList decomp; decompose @@ -336,7 +357,7 @@ Foam::labelList Foam::decompositionMethods::parMetis::decompose cellCells.m(), points, nWeights, - scaleWeights(pointWeights, nWeights), + intWeights, labelList(), decomp ); @@ -382,7 +403,8 @@ Foam::labelList Foam::decompositionMethods::parMetis::decompose cellCells ); - const label nWeights = this->nWeights(regionPoints, pointWeights); + label nWeights = this->nWeights(regionPoints, pointWeights); + const labelList intWeights(scaleWeights(pointWeights, nWeights)); // Decompose using weights labelList decomp; @@ -392,7 +414,7 @@ Foam::labelList Foam::decompositionMethods::parMetis::decompose cellCells.offsets(), regionPoints, nWeights, - scaleWeights(pointWeights, nWeights), + intWeights, labelList(), decomp ); @@ -437,7 +459,8 @@ Foam::labelList Foam::decompositionMethods::parMetis::decompose // xadj(celli) : start of information in adjncy for celli CompactListList