zhyang-dev/OpenFOAM-12
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

decompositionMethods: Standardised the conversion of scalar weights to integer weights

Browse Source 
and general rationalisation and cleanup
This commit is contained in:
Henry Weller
2024-05-21 11:24:34 +01:00
parent c437265c80
commit 30afcdc331
18 changed files with 228 additions and 496 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
src/parallel/decompose/decompositionMethods/decompositionMethod/decompositionMethod.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2011-2022 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2024 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -249,14 +249,14 @@ Foam::labelList Foam::decompositionMethod::decompose
const pointField& coarsePoints
)
{
scalarField cWeights(coarsePoints.size(), 1.0);
scalarField cellWeights(coarsePoints.size(), 1.0);
return decompose
(
mesh,
fineToCoarse,
coarsePoints,
cWeights
cellWeights
);
}
@ -264,12 +264,43 @@ Foam::labelList Foam::decompositionMethod::decompose
Foam::labelList Foam::decompositionMethod::decompose
(
const labelListList& globalCellCells,
const pointField& cc
const pointField& cellCentres
)
{
scalarField cWeights(cc.size(), 1.0);
scalarField cellWeights(cellCentres.size(), 1.0);
return decompose(globalCellCells, cc, cWeights);
return decompose(globalCellCells, cellCentres, cellWeights);
}
Foam::labelList Foam::decompositionMethod::scaleWeights
(
const scalarField& weights,
const label nWeights,
const bool distributed
)
{
labelList intWeights;
if (weights.size() > 0)
{
const scalar sumWeights
(
distributed
? gSum(weights)
: sum(weights)
);
const scalar scale = labelMax/(2*sumWeights);
// Convert to integers.
intWeights.setSize(weights.size());
forAll(intWeights, i)
{
intWeights[i] = ceil(scale*weights[i]);
}
}
return intWeights;
}

12
src/parallel/decompose/decompositionMethods/decompositionMethod/decompositionMethod.H
View File

@ -53,6 +53,7 @@ protected:
// Protected data
dictionary decompositionDict_;
label nProcessors_;
//- Optional constraints
@ -214,6 +215,15 @@ public:
// Other
//- Convert the given scalar weights to labels
// in the range 0-labelMax/2
static labelList scaleWeights
(
const scalarField& weights,
const label nWeights,
const bool distributed = true
);
//- Helper: determine (local or global) cellCells from mesh
// agglomeration. Agglomeration is local to the processor.
// local : connections are in local indices. Coupled across
@ -301,7 +311,7 @@ public:
labelList decompose
(
const polyMesh& mesh,
const scalarField& cWeights
const scalarField& cellWeights
);

22
src/parallel/decompose/decompositionMethods/hierarchGeomDecomp/hierarchGeomDecomp.H
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2011-2021 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2024 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -194,18 +194,22 @@ public:
virtual labelList decompose
(
const polyMesh& mesh,
const pointField& cc,
const scalarField& cWeights
const pointField& cellCentres,
const scalarField& cellWeights
)
{
return decompose(cc, cWeights);
return decompose(cellCentres, cellWeights);
}
//- Without weights. Code for weighted decomposition is a bit complex
// so kept separate for now.
virtual labelList decompose(const polyMesh& mesh, const pointField& cc)
virtual labelList decompose
(
const polyMesh& mesh,
const pointField& cellCentres
)
{
return decompose(cc);
return decompose(cellCentres);
}
//- Return for every coordinate the wanted processor number. Explicitly
@ -218,11 +222,11 @@ public:
virtual labelList decompose
(
const labelListList& globalCellCells,
const pointField& cc,
const scalarField& cWeights
const pointField& cellCentres,
const scalarField& cellWeights
)
{
return decompose(cc, cWeights);
return decompose(cellCentres, cellWeights);
}
virtual labelList decompose

10
src/parallel/decompose/decompositionMethods/manualDecomp/manualDecomp.H
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2011-2021 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2024 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -80,8 +80,8 @@ public:
virtual labelList decompose
(
const polyMesh& mesh,
const pointField& cc,
const scalarField& cWeights
const pointField& cellCentres,
const scalarField& cellWeights
);
//- Return for every coordinate the wanted processor number. Explicitly
@ -94,8 +94,8 @@ public:
virtual labelList decompose
(
const labelListList& globalCellCells,
const pointField& cc,
const scalarField& cWeights
const pointField& cellCentres,
const scalarField& cellWeights
)
{
NotImplemented;

42
src/parallel/decompose/decompositionMethods/multiLevelDecomp/multiLevelDecomp.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2011-2023 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2024 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -136,8 +136,7 @@ void Foam::multiLevelDecomp::decompose
const scalarField& pointWeights,
const labelList& pointMap, // map back to original points
const label levelI,
labelField& finalDecomp
labelField& decomp
)
{
labelList dist
@ -153,7 +152,7 @@ void Foam::multiLevelDecomp::decompose
forAll(pointMap, i)
{
label orig = pointMap[i];
finalDecomp[orig] += dist[i];
decomp[orig] += dist[i];
}
if (levelI != methods_.size()-1)
@ -165,7 +164,7 @@ void Foam::multiLevelDecomp::decompose
labelListList domainToPoints(invertOneToMany(n, dist));
// 'Make space' for new levels of decomposition
finalDecomp *= methods_[levelI+1].nDomains();
decomp *= methods_[levelI+1].nDomains();
// Extract processor+local index from point-point addressing
if (debug && Pstream::master())
@ -233,7 +232,7 @@ void Foam::multiLevelDecomp::decompose
subPointMap,
levelI+1,
finalDecomp
decomp
);
if (debug && Pstream::master())
{
@ -374,28 +373,35 @@ Foam::multiLevelDecomp::multiLevelDecomp(const dictionary& decompositionDict)
Foam::labelList Foam::multiLevelDecomp::decompose
(
const polyMesh& mesh,
const pointField& cc,
const scalarField& cWeights
const pointField& cellCentres,
const scalarField& cellWeights
)
{
CompactListList<label> cellCells;
calcCellCells(mesh, identityMap(cc.size()), cc.size(), true, cellCells);
calcCellCells
(
mesh,
identityMap(cellCentres.size()),
cellCentres.size(),
true,
cellCells
);
labelField finalDecomp(cc.size(), 0);
labelList cellMap(identityMap(cc.size()));
labelField decomp(cellCentres.size(), 0);
labelList cellMap(identityMap(cellCentres.size()));
decompose
(
cellCells.list(),
cc,
cWeights,
cellCentres,
cellWeights,
cellMap, // map back to original cells
0,
finalDecomp
decomp
);
return finalDecomp;
return decomp;
}
@ -406,7 +412,7 @@ Foam::labelList Foam::multiLevelDecomp::decompose
const scalarField& pointWeights
)
{
labelField finalDecomp(points.size(), 0);
labelField decomp(points.size(), 0);
labelList pointMap(identityMap(points.size()));
decompose
@ -417,10 +423,10 @@ Foam::labelList Foam::multiLevelDecomp::decompose
pointMap, // map back to original points
0,
finalDecomp
decomp
);
return finalDecomp;
return decomp;
}

6
src/parallel/decompose/decompositionMethods/multiLevelDecomp/multiLevelDecomp.H
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2011-2021 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2024 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -123,8 +123,8 @@ public:
virtual labelList decompose
(
const labelListList& globalCellCells,
const pointField& cc,
const scalarField& cWeights
const pointField& cellCentres,
const scalarField& cellWeights
);

12
src/parallel/decompose/decompositionMethods/noDecomp/noDecomp.H
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2012-2021 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2012-2024 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -75,11 +75,11 @@ public:
virtual labelList decompose
(
const polyMesh& mesh,
const pointField& cc,
const scalarField& cWeights
const pointField& cellCentres,
const scalarField& cellWeights
)
{
return labelList(cc.size(), Pstream::myProcNo());
return labelList(cellCentres.size(), Pstream::myProcNo());
}
//- Return for every coordinate the wanted processor number. Explicitly
@ -92,8 +92,8 @@ public:
virtual labelList decompose
(
const labelListList& globalCellCells,
const pointField& cc,
const scalarField& cWeights
const pointField& cellCentres,
const scalarField& cellWeights
)
{
return labelList(globalCellCells.size(), Pstream::myProcNo());

10
src/parallel/decompose/decompositionMethods/randomDecomp/randomDecomp.H
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2022 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2022-2024 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -83,8 +83,8 @@ public:
virtual labelList decompose
(
const polyMesh& mesh,
const pointField& cc,
const scalarField& cWeights
const pointField& cellCentres,
const scalarField& cellWeights
);
//- Return for every coordinate the wanted processor number. Explicitly
@ -92,8 +92,8 @@ public:
virtual labelList decompose
(
const labelListList& globalCellCells,
const pointField& cc,
const scalarField& cWeights
const pointField& cellCentres,
const scalarField& cellWeights
)
{
NotImplemented;

8
src/parallel/decompose/decompositionMethods/simpleGeomDecomp/simpleGeomDecomp.H
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2011-2021 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2024 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -114,11 +114,11 @@ public:
virtual labelList decompose
(
const labelListList& globalCellCells,
const pointField& cc,
const scalarField& cWeights
const pointField& cellCentres,
const scalarField& cellWeights
)
{
return decompose(cc, cWeights);
return decompose(cellCentres, cellWeights);
}

12
src/parallel/decompose/decompositionMethods/structuredDecomp/structuredDecomp.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2011-2021 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2024 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -69,8 +69,8 @@ Foam::structuredDecomp::structuredDecomp(const dictionary& decompositionDict)
Foam::labelList Foam::structuredDecomp::decompose
(
const polyMesh& mesh,
const pointField& cc,
const scalarField& cWeights
const pointField& cellCentres,
const scalarField& cellWeights
)
{
const polyBoundaryMesh& pbm = mesh.boundaryMesh();
@ -97,15 +97,15 @@ Foam::labelList Foam::structuredDecomp::decompose
fvMeshSubset subsetter(dynamic_cast<const fvMesh&>(mesh));
subsetter.setLargeCellSubset(patchCells);
const fvMesh& subMesh = subsetter.subMesh();
pointField subCc(cc, subsetter.cellMap());
scalarField subWeights(cWeights, subsetter.cellMap());
pointField subCc(cellCentres, subsetter.cellMap());
scalarField subWeights(cellWeights, subsetter.cellMap());
// Decompose the layer of cells
labelList subDecomp(method_().decompose(subMesh, subCc, subWeights));
// Transfer to final decomposition
labelList finalDecomp(cc.size(), -1);
labelList finalDecomp(cellCentres.size(), -1);
forAll(subDecomp, i)
{
finalDecomp[subsetter.cellMap()[i]] = subDecomp[i];

6
src/parallel/decompose/decompositionMethods/structuredDecomp/structuredDecomp.H
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2011-2021 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2024 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -93,8 +93,8 @@ public:
virtual labelList decompose
(
const labelListList& globalCellCells,
const pointField& cc,
const scalarField& cWeights
const pointField& cellCentres,
const scalarField& cellWeights
);

105
src/parallel/decompose/metisDecomp/metisDecomp.C
View File

@ -47,61 +47,41 @@ namespace Foam
Foam::label Foam::metisDecomp::decompose
(
const List<label>& adjncy,
const List<label>& xadj,
const scalarField& cWeights,
const labelList& adjncy,
const labelList& xadj,
const scalarField& cellWeights,
List<label>& finalDecomp
labelList& decomp
)
{
// Method of decomposition
// recursive: multi-level recursive bisection (default)
// k-way: multi-level k-way
// kWay: multi-level k-way
word method("recursive");
label numCells = xadj.size()-1;
// Decomposition options
List<label> options(METIS_NOPTIONS);
labelList options(METIS_NOPTIONS);
METIS_SetDefaultOptions(options.begin());
// Processor weights initialised with no size, only used if specified in
// a file
Field<real_t> processorWeights;
if (cellWeights.size() > 0 && cellWeights.size() != numCells)
{
FatalErrorInFunction
<< "Number of cell weights " << cellWeights.size()
<< " does not equal number of cells " << numCells
<< exit(FatalError);
}
// Cell weights (so on the vertices of the dual)
List<label> cellWeights;
labelList intWeights(scaleWeights(cellWeights, 1, false));
// Face weights (so on the edges of the dual)
List<label> faceWeights;
// Check for externally provided cellweights and if so initialise weights
scalar minWeights = gMin(cWeights);
if (cWeights.size() > 0)
{
if (minWeights <= 0)
{
WarningInFunction
<< "Illegal minimum weight " << minWeights
<< endl;
}
if (cWeights.size() != numCells)
{
FatalErrorInFunction
<< "Number of cell weights " << cWeights.size()
<< " does not equal number of cells " << numCells
<< exit(FatalError);
}
// Convert to integers.
cellWeights.setSize(cWeights.size());
forAll(cellWeights, i)
{
cellWeights[i] = label(cWeights[i]/minWeights);
}
}
labelList faceWeights;
// Check for user supplied weights and decomp options
@ -110,17 +90,15 @@ Foam::label Foam::metisDecomp::decompose
const dictionary& metisCoeffs =
decompositionDict_.subDict("metisCoeffs");
word weightsFile;
if (metisCoeffs.readIfPresent("method", method))
{
if (method != "recursive" && method != "k-way")
if (method != "recursive" && method != "kWay")
{
FatalErrorInFunction
FatalIOErrorInFunction(metisCoeffs)
<< "Method " << method << " in metisCoeffs in dictionary : "
<< decompositionDict_.name()
<< " should be 'recursive' or 'k-way'"
<< exit(FatalError);
<< " should be 'recursive' or 'kWay'"
<< exit(FatalIOError);
}
Info<< "metisDecomp : Using Metis method " << method
@ -131,11 +109,11 @@ Foam::label Foam::metisDecomp::decompose
{
if (options.size() != METIS_NOPTIONS)
{
FatalErrorInFunction
FatalIOErrorInFunction(metisCoeffs)
<< "Number of options in metisCoeffs dictionary : "
<< decompositionDict_.name()
<< " should be " << METIS_NOPTIONS << " found " << options
<< exit(FatalError);
<< exit(FatalIOError);
}
Info<< "Using Metis options " << options << nl << endl;
@ -147,11 +125,11 @@ Foam::label Foam::metisDecomp::decompose
if (processorWeights.size() != nProcessors_)
{
FatalErrorInFunction
FatalIOErrorInFunction(metisCoeffs)
<< "Number of processor weights "
<< processorWeights.size()
<< " does not equal number of domains " << nProcessors_
<< exit(FatalError);
<< exit(FatalIOError);
}
}
}
@ -160,7 +138,7 @@ Foam::label Foam::metisDecomp::decompose
label nProcs = nProcessors_;
// Output: cell -> processor addressing
finalDecomp.setSize(numCells);
decomp.setSize(numCells);
// Output: number of cut edges
label edgeCut = 0;
@ -171,9 +149,9 @@ Foam::label Foam::metisDecomp::decompose
(
&numCells, // num vertices in graph
&ncon, // num balancing constraints
const_cast<List<label>&>(xadj).begin(), // indexing into adjncy
const_cast<List<label>&>(adjncy).begin(), // neighbour info
cellWeights.begin(),// vertexweights
const_cast<labelList&>(xadj).begin(), // indexing into adjncy
const_cast<labelList&>(adjncy).begin(), // neighbour info
intWeights.begin(),// vertexweights
nullptr, // vsize: total communication vol
faceWeights.begin(),// edgeweights
&nProcs, // nParts
@ -181,7 +159,7 @@ Foam::label Foam::metisDecomp::decompose
nullptr, // ubvec: processor imbalance (default)
options.begin(),
&edgeCut,
finalDecomp.begin()
decomp.begin()
);
}
else
@ -190,9 +168,9 @@ Foam::label Foam::metisDecomp::decompose
(
&numCells, // num vertices in graph
&ncon, // num balancing constraints
const_cast<List<label>&>(xadj).begin(), // indexing into adjncy
const_cast<List<label>&>(adjncy).begin(), // neighbour info
cellWeights.begin(),// vertexweights
const_cast<labelList&>(xadj).begin(), // indexing into adjncy
const_cast<labelList&>(adjncy).begin(), // neighbour info
intWeights.begin(),// vertexweights
nullptr, // vsize: total communication vol
faceWeights.begin(),// edgeweights
&nProcs, // nParts
@ -200,7 +178,7 @@ Foam::label Foam::metisDecomp::decompose
nullptr, // ubvec: processor imbalance (default)
options.begin(),
&edgeCut,
finalDecomp.begin()
decomp.begin()
);
}
@ -236,6 +214,9 @@ Foam::labelList Foam::metisDecomp::decompose
<< exit(FatalError);
}
// Make Metis CSR (Compressed Storage Format) storage
// adjncy : contains neighbours (= edges in graph)
// xadj(celli) : start of information in adjncy for celli
CompactListList<label> cellCells;
calcCellCells
(
@ -248,7 +229,13 @@ Foam::labelList Foam::metisDecomp::decompose
// Decompose using default weights
labelList decomp;
decompose(cellCells.m(), cellCells.offsets(), pointWeights, decomp);
decompose
(
cellCells.m(),
cellCells.offsets(),
pointWeights,
decomp
);
return decomp;
}
@ -278,8 +265,8 @@ Foam::labelList Foam::metisDecomp::decompose
calcCellCells(mesh, cellToRegion, regionPoints.size(), false, cellCells);
// Decompose using default weights
labelList finalDecomp;
decompose(cellCells.m(), cellCells.offsets(), regionWeights, finalDecomp);
labelList decomp;
decompose(cellCells.m(), cellCells.offsets(), regionWeights, decomp);
// Rework back into decomposition for original mesh
@ -287,7 +274,7 @@ Foam::labelList Foam::metisDecomp::decompose
forAll(fineDistribution, i)
{
fineDistribution[i] = finalDecomp[cellToRegion[i]];
fineDistribution[i] = decomp[cellToRegion[i]];
}
return fineDistribution;

6
src/parallel/decompose/metisDecomp/metisDecomp.H
View File

@ -53,10 +53,10 @@ class metisDecomp
//- Call Metis with options from dictionary.
label decompose
(
const List<label>& adjncy,
const List<label>& xadj,
const labelList& adjncy,
const labelList& xadj,
const scalarField& cellWeights,
List<label>& finalDecomp
labelList& finalDecomp
);

273
src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C
View File

@ -77,198 +77,6 @@ void Foam::ptscotchDecomp::check(const int retVal, const char* str)
}
////- Does prevention of 0 cell domains and calls ptscotch.
//Foam::label Foam::ptscotchDecomp::decomposeZeroDomains
//(
// const fileName& meshPath,
// const labelList& initadjncy,
// const labelList& initxadj,
// const scalarField& initcellWeights,
//
// labelList& decomp
//) const
//{
// globalIndex globalCells(initxadj.size()-1);
//
// bool hasZeroDomain = false;
// for (label proci = 0; proci < Pstream::nProcs(); proci++)
// {
// if (globalCells.localSize(proci) == 0)
// {
// hasZeroDomain = true;
// break;
// }
// }
//
// if (!hasZeroDomain)
// {
// return decompose
// (
// meshPath,
// initadjncy,
// initxadj,
// initcellWeights,
// decomp
// );
// }
//
//
// if (debug)
// {
// Info<< "ptscotchDecomp : have graphs with locally 0 cells."
// << " trickling down." << endl;
// }
//
// // Make sure every domain has at least one cell
// // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// // (scotch does not like zero sized domains)
// // Trickle cells from processors that have them up to those that
// // don't.
//
//
// // Number of cells to send to the next processor
// // (is same as number of cells next processor has to receive)
// labelList nSendCells(Pstream::nProcs(), 0);
//
// for (label proci = nSendCells.size()-1; proci >=1; proci--)
// {
// label nLocalCells = globalCells.localSize(proci);
// if (nLocalCells-nSendCells[proci] < 1)
// {
// nSendCells[proci-1] = nSendCells[proci]-nLocalCells+1;
// }
// }
//
// // First receive (so increasing the sizes of all arrays)
//
// Field<int> xadj(initxadj);
// Field<int> adjncy(initadjncy);
// scalarField cellWeights(initcellWeights);
//
// if (Pstream::myProcNo() >= 1 && nSendCells[Pstream::myProcNo()-1] > 0)
// {
// // Receive cells from previous processor
// IPstream fromPrevProc(Pstream::commsTypes::blocking,
// Pstream::myProcNo()-1);
//
// Field<int> prevXadj(fromPrevProc);
// Field<int> prevAdjncy(fromPrevProc);
// scalarField prevCellWeights(fromPrevProc);
//
// if (prevXadj.size() != nSendCells[Pstream::myProcNo()-1])
// {
// FatalErrorInFunction
// << "Expected from processor " << Pstream::myProcNo()-1
// << " connectivity for " << nSendCells[Pstream::myProcNo()-1]
// << " nCells but only received " << prevXadj.size()
// << abort(FatalError);
// }
//
// // Insert adjncy
// prepend(prevAdjncy, adjncy);
// // Adapt offsets and prepend xadj
// xadj += prevAdjncy.size();
// prepend(prevXadj, xadj);
// // Weights
// prepend(prevCellWeights, cellWeights);
// }
//
//
// // Send to my next processor
//
// if (nSendCells[Pstream::myProcNo()] > 0)
// {
// // Send cells to next processor
// OPstream toNextProc(Pstream::commsTypes::blocking,
// Pstream::myProcNo()+1);
//
// label nCells = nSendCells[Pstream::myProcNo()];
// label startCell = xadj.size()-1 - nCells;
// label startFace = xadj[startCell];
// label nFaces = adjncy.size()-startFace;
//
// // Send for all cell data: last nCells elements
// // Send for all face data: last nFaces elements
// toNextProc
// << Field<int>::subField(xadj, nCells, startCell)-startFace
// << Field<int>::subField(adjncy, nFaces, startFace)
// <<
// (
// cellWeights.size()
// ? static_cast<const scalarField&>
// (
// scalarField::subField(cellWeights, nCells, startCell)
// )
// : scalarField(0)
// );
//
// // Remove data that has been sent
// if (cellWeights.size())
// {
// cellWeights.setSize(cellWeights.size()-nCells);
// }
// adjncy.setSize(adjncy.size()-nFaces);
// xadj.setSize(xadj.size() - nCells);
// }
//
//
// // Do decomposition as normal. Sets decomp.
// label result = decompose(meshPath, adjncy, xadj, cellWeights, decomp);
//
//
// if (debug)
// {
// Info<< "ptscotchDecomp : have graphs with locally 0 cells."
// << " trickling up." << endl;
// }
//
//
// // If we sent cells across make sure we undo it
// // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
//
// // Receive back from next processor if I sent something
// if (nSendCells[Pstream::myProcNo()] > 0)
// {
// IPstream fromNextProc(Pstream::commsTypes::blocking,
// Pstream::myProcNo()+1);
//
// labelList nextFinalDecomp(fromNextProc);
//
// if (nextFinalDecomp.size() != nSendCells[Pstream::myProcNo()])
// {
// FatalErrorInFunction
// << "Expected from processor " << Pstream::myProcNo()+1
// << " decomposition for " << nSendCells[Pstream::myProcNo()]
// << " nCells but only received " << nextFinalDecomp.size()
// << abort(FatalError);
// }
//
// append(nextFinalDecomp, decomp);
// }
//
// // Send back to previous processor.
// if (Pstream::myProcNo() >= 1 && nSendCells[Pstream::myProcNo()-1] > 0)
// {
// OPstream toPrevProc(Pstream::commsTypes::blocking,
// Pstream::myProcNo()-1);
//
// label nToPrevious = nSendCells[Pstream::myProcNo()-1];
//
// toPrevProc <<
// SubList<label>
// (
// decomp,
// nToPrevious,
// decomp.size()-nToPrevious
// );
//
// // Remove locally what has been sent
// decomp.setSize(decomp.size()-nToPrevious);
// }
// return result;
//}
Foam::label Foam::ptscotchDecomp::decompose
(
const fileName& meshPath,
@ -279,8 +87,7 @@ Foam::label Foam::ptscotchDecomp::decompose
) const
{
labelList dummyAdjncy(1);
labelList dummyXadj(1);
dummyXadj[0] = 0;
labelList dummyXadj(1, 0);
return decompose
(
@ -303,7 +110,6 @@ Foam::label Foam::ptscotchDecomp::decompose
const label xadjSize,
const label xadj[],
const scalarField& cellWeights,
labelList& decomp
) const
{
@ -378,7 +184,6 @@ Foam::label Foam::ptscotchDecomp::decompose
const dictionary& scotchCoeffs =
decompositionDict_.subDict("scotchCoeffs");
string strategy;
if (scotchCoeffs.readIfPresent("strategy", strategy))
{
@ -387,9 +192,6 @@ Foam::label Foam::ptscotchDecomp::decompose
Info<< "ptscotchDecomp : Using strategy " << strategy << endl;
}
SCOTCH_stratDgraphMap(&stradat, strategy.c_str());
// fprintf(stdout, "S\tStrat=");
// SCOTCH_stratSave(&stradat, stdout);
// fprintf(stdout, "\n");
}
}
@ -399,21 +201,9 @@ Foam::label Foam::ptscotchDecomp::decompose
labelList velotab;
// Check for externally provided cellweights and if so initialise weights
scalar minWeights = gMin(cellWeights);
scalar maxWeights = gMax(cellWeights);
if (maxWeights > minWeights)
if (returnReduce(cellWeights.size(), sumOp<label>()))
{
if (minWeights <= 0)
{
WarningInFunction
<< "Illegal minimum weight " << minWeights
<< endl;
}
if (cellWeights.size() != xadjSize-1)
{
FatalErrorInFunction
@ -421,58 +211,24 @@ Foam::label Foam::ptscotchDecomp::decompose
<< " does not equal number of cells " << xadjSize-1
<< exit(FatalError);
}
}
scalar velotabSum = gSum(cellWeights)/minWeights;
velotab = scaleWeights(cellWeights, 1);
scalar rangeScale(1.0);
if (Pstream::master())
{
if (velotabSum > scalar(labelMax - 1))
{
// 0.9 factor of safety to avoid floating point round-off in
// rangeScale tipping the subsequent sum over the integer limit.
rangeScale = 0.9*scalar(labelMax - 1)/velotabSum;
WarningInFunction
<< "Sum of weights has overflowed integer: " << velotabSum
<< ", compressing weight scale by a factor of " << rangeScale
<< endl;
}
}
Pstream::scatter(rangeScale);
if (maxWeights > minWeights)
{
if (cellWeights.size())
{
// Convert to integers.
velotab.setSize(cellWeights.size());
forAll(velotab, i)
{
velotab[i] =
label((cellWeights[i]/minWeights - 1)*rangeScale) + 1;
}
}
else
if (!cellWeights.size())
{
// Locally zero cells but not globally. Make sure we have
// some size so .begin() does not return null pointer. Data
// itself is never used.
velotab.setSize(1);
velotab[0] = 1;
velotab.setSize(1, 1);
}
}
if (debug)
{
Pout<< "SCOTCH_dgraphInit" << endl;
}
SCOTCH_Dgraph grafdat;
check
(
@ -533,6 +289,7 @@ Foam::label Foam::ptscotchDecomp::decompose
{
Pout<< "SCOTCH_archInit" << endl;
}
SCOTCH_Arch archdat;
check(SCOTCH_archInit(&archdat), "SCOTCH_archInit");
@ -607,21 +364,6 @@ Foam::label Foam::ptscotchDecomp::decompose
feenableexcept(oldExcepts);
#endif
// decomp.setSize(xadjSize-1);
// check
//(
// SCOTCH_dgraphPart
// (
// &grafdat,
// nProcessors_, // partnbr
// &stradat, // const SCOTCH_Strat *
// decomp.begin() // parttab
// ),
// "SCOTCH_graphPart"
//);
if (debug)
{
Pout<< "SCOTCH_dgraphExit" << endl;
@ -769,7 +511,6 @@ Foam::labelList Foam::ptscotchDecomp::decompose
// Make Metis CSR (Compressed Storage Format) storage
// adjncy : contains neighbours (= edges in graph)
// xadj(celli) : start of information in adjncy for celli
CompactListList<label> cellCells(globalCellCells);
// Decompose using weights

12
src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.H
View File

@ -75,8 +75,8 @@ class ptscotchDecomp
const fileName& meshPath,
const labelList& adjncy,
const labelList& xadj,
const scalarField& cWeights,
labelList& finalDecomp
const scalarField& cellWeights,
labelList& decomp
) const;
//- Low level decompose
@ -87,8 +87,8 @@ class ptscotchDecomp
const label adjncy[],
const label xadjSize,
const label xadj[],
const scalarField& cWeights,
labelList& finalDecomp
const scalarField& cellWeights,
labelList& decomp
) const;
@ -149,8 +149,8 @@ public:
virtual labelList decompose
(
const labelListList& globalCellCells,
const pointField& cc,
const scalarField& cWeights
const pointField& cellCentres,
const scalarField& cellWeights
);

115
src/parallel/decompose/scotchDecomp/scotchDecomp.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2011-2023 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2024 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -85,11 +85,10 @@ void Foam::scotchDecomp::check(const int retVal, const char* str)
Foam::label Foam::scotchDecomp::decompose
(
const fileName& meshPath,
const List<label>& adjncy,
const List<label>& xadj,
const scalarField& cWeights,
List<label>& finalDecomp
const labelList& adjncy,
const labelList& xadj,
const scalarField& cellWeights,
labelList& decomp
)
{
if (!Pstream::parRun())
@ -99,8 +98,8 @@ Foam::label Foam::scotchDecomp::decompose
meshPath,
adjncy,
xadj,
cWeights,
finalDecomp
cellWeights,
decomp
);
}
else
@ -122,10 +121,10 @@ Foam::label Foam::scotchDecomp::decompose
// Insert my own
label nTotalCells = 0;
forAll(cWeights, celli)
forAll(cellWeights, celli)
{
allXadj[nTotalCells] = xadj[celli];
allWeights[nTotalCells++] = cWeights[celli];
allWeights[nTotalCells++] = cellWeights[celli];
}
nTotalConnections = 0;
forAll(adjncy, i)
@ -179,7 +178,7 @@ Foam::label Foam::scotchDecomp::decompose
);
}
// Get my own part (always first)
finalDecomp = SubField<label>
decomp = SubField<label>
(
allFinalDecomp,
globalCells.localSize()
@ -195,7 +194,7 @@ Foam::label Foam::scotchDecomp::decompose
Pstream::masterNo()
);
toMaster<< adjncy << SubField<label>(xadj, xadj.size()-1)
<< cWeights;
<< cellWeights;
}
// Receive back decomposition
@ -204,22 +203,21 @@ Foam::label Foam::scotchDecomp::decompose
Pstream::commsTypes::scheduled,
Pstream::masterNo()
);
fromMaster >> finalDecomp;
fromMaster >> decomp;
}
}
return 0;
}
// Call scotch with options from dictionary.
Foam::label Foam::scotchDecomp::decomposeOneProc
(
const fileName& meshPath,
const List<label>& adjncy,
const List<label>& xadj,
const scalarField& cWeights,
const labelList& adjncy,
const labelList& xadj,
const scalarField& cellWeights,
List<label>& finalDecomp
labelList& decomp
)
{
// Dump graph
@ -295,56 +293,23 @@ Foam::label Foam::scotchDecomp::decomposeOneProc
// Graph
// ~~~~~
List<label> velotab;
labelList velotab;
// Check for externally provided cellweights and if so initialise weights
// Note: min, not gMin since routine runs on master only.
scalar minWeights = min(cWeights);
if (!cWeights.empty())
if (!cellWeights.empty())
{
if (minWeights <= 0)
{
WarningInFunction
<< "Illegal minimum weight " << minWeights
<< endl;
}
if (cWeights.size() != xadj.size()-1)
if (cellWeights.size() != xadj.size()-1)
{
FatalErrorInFunction
<< "Number of cell weights " << cWeights.size()
<< "Number of cell weights " << cellWeights.size()
<< " does not equal number of cells " << xadj.size()-1
<< exit(FatalError);
}
scalar velotabSum = sum(cWeights)/minWeights;
scalar rangeScale(1.0);
if (velotabSum > scalar(labelMax - 1))
{
// 0.9 factor of safety to avoid floating point round-off in
// rangeScale tipping the subsequent sum over the integer limit.
rangeScale = 0.9*scalar(labelMax - 1)/velotabSum;
WarningInFunction
<< "Sum of weights has overflowed integer: " << velotabSum
<< ", compressing weight scale by a factor of " << rangeScale
<< endl;
}
// Convert to integers.
velotab.setSize(cWeights.size());
forAll(velotab, i)
{
velotab[i] = int((cWeights[i]/minWeights - 1)*rangeScale) + 1;
}
velotab = scaleWeights(cellWeights, 1, false);
}
SCOTCH_Graph grafdat;
check(SCOTCH_graphInit(&grafdat), "SCOTCH_graphInit");
check
@ -374,7 +339,7 @@ Foam::label Foam::scotchDecomp::decomposeOneProc
SCOTCH_Arch archdat;
check(SCOTCH_archInit(&archdat), "SCOTCH_archInit");
List<label> processorWeights;
labelList processorWeights;
if (decompositionDict_.found("scotchCoeffs"))
{
const dictionary& scotchCoeffs =
@ -414,8 +379,8 @@ Foam::label Foam::scotchDecomp::decomposeOneProc
);
#endif
finalDecomp.setSize(xadj.size()-1);
finalDecomp = 0;
decomp.setSize(xadj.size()-1);
decomp = 0;
check
(
SCOTCH_graphMap
@ -423,7 +388,7 @@ Foam::label Foam::scotchDecomp::decomposeOneProc
&grafdat,
&archdat,
&stradat, // const SCOTCH_Strat *
finalDecomp.begin() // parttab
decomp.begin() // parttab
),
"SCOTCH_graphMap"
);
@ -485,22 +450,16 @@ Foam::labelList Foam::scotchDecomp::decompose
);
// Decompose using default weights
List<label> finalDecomp;
labelList decomp;
decompose
(
mesh.time().path()/mesh.name(),
cellCells.m(),
cellCells.offsets(),
pointWeights,
finalDecomp
decomp
);
// Copy back to labelList
labelList decomp(finalDecomp.size());
forAll(decomp, i)
{
decomp[i] = finalDecomp[i];
}
return decomp;
}
@ -533,14 +492,14 @@ Foam::labelList Foam::scotchDecomp::decompose
);
// Decompose using weights
List<label> finalDecomp;
labelList decomp;
decompose
(
mesh.time().path()/mesh.name(),
cellCells.m(),
cellCells.offsets(),
pointWeights,
finalDecomp
decomp
);
// Rework back into decomposition for original mesh_
@ -548,7 +507,7 @@ Foam::labelList Foam::scotchDecomp::decompose
forAll(fineDistribution, i)
{
fineDistribution[i] = finalDecomp[agglom[i]];
fineDistribution[i] = decomp[agglom[i]];
}
return fineDistribution;
@ -559,7 +518,7 @@ Foam::labelList Foam::scotchDecomp::decompose
(
const labelListList& globalCellCells,
const pointField& cellCentres,
const scalarField& cWeights
const scalarField& cellWeights
)
{
if (cellCentres.size() != globalCellCells.size())
@ -578,22 +537,16 @@ Foam::labelList Foam::scotchDecomp::decompose
CompactListList<label> cellCells(globalCellCells);
// Decompose using weights
List<label> finalDecomp;
labelList decomp;
decompose
(
"scotch",
cellCells.m(),
cellCells.offsets(),
cWeights,
finalDecomp
cellWeights,
decomp
);
// Copy back to labelList
labelList decomp(finalDecomp.size());
forAll(decomp, i)
{
decomp[i] = finalDecomp[i];
}
return decomp;
}

22
src/parallel/decompose/scotchDecomp/scotchDecomp.H
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2011-2022 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2024 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -233,20 +233,20 @@ class scotchDecomp
label decompose
(
const fileName& meshPath,
const List<label>& adjncy,
const List<label>& xadj,
const scalarField& cWeights,
List<label>& finalDecomp
const labelList& adjncy,
const labelList& xadj,
const scalarField& cellWeights,
labelList& decomp
);
//- Decompose non-parallel
label decomposeOneProc
(
const fileName& meshPath,
const List<label>& adjncy,
const List<label>& xadj,
const scalarField& cWeights,
List<label>& finalDecomp
const labelList& adjncy,
const labelList& xadj,
const scalarField& cellWeights,
labelList& decomp
);
@ -311,8 +311,8 @@ public:
virtual labelList decompose
(
const labelListList& globalCellCells,
const pointField& cc,
const scalarField& cWeights
const pointField& cellCentres,
const scalarField& cellWeights
);

8
src/parallel/decompose/zoltanDecomp/zoltanDecomp.H
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2021 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2021-2024 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -101,7 +101,7 @@ class zoltanDecomp
(
const CompactListList<label>& adjacency,
const pointField& points,
const scalarField& cWeights,
const scalarField& cellWeights,
List<label>& finalDecomp
) const;
@ -163,8 +163,8 @@ public:
virtual labelList decompose
(
const labelListList& globalCellCells,
const pointField& cc,
const scalarField& cWeights
const pointField& cellCentres,
const scalarField& cellWeights
);