zhyang-dev/OpenFOAM-12
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Lagrangian: Added support for automatic run-time load-balancing

Browse Source 
Optional CPU load caching can be switched-on for Lagrangian cloud tracking
and/or chemistry integration using the new cpuLoad switch in the cloudProperties
or chemistryProperties dictionary files respectively and used for
multi-constraint load-balancing by the fvMeshDistributorsLoadBalancer specified
in the dynamicMeshDict file

distributor
{
    type            loadBalancer;

    libs            ("libfvMeshDistributors.so");

    multiConstraint true;
    redistributionInterval  10;
}

which used the distributor specified in the decomposeParDict file, e.g.

numberOfSubdomains 12;

decomposer      simple;
distributor     zoltan;
libs            ("libzoltanDecomp.so");

simpleCoeffs
{
    n           (2 2 3);
}

zoltanCoeffs
{
    lb_method   rcb;
}

The incompressibleDenseParticleFluid/cyclone case has been updated to
demonstrate this new functionality and shows a speedup ~50% using the Zoltan RCB
multi-constraint distributor.  The multicomponentFluid/counterFlowFlame2D_GRI
case has also been updated to use the new cpuLoad switch.
This commit is contained in:
Henry Weller
2024-05-16 13:46:20 +01:00
parent 476bb42b04
commit 41705e9eca
21 changed files with 252 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
src/finiteVolume/fvMesh/fvMeshDistributors/cpuLoad/cpuLoad.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2022 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2022-2024 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -107,6 +107,24 @@ Foam::optionalCpuLoad& Foam::optionalCpuLoad::New
}
Foam::optionalCpuLoad& Foam::optionalCpuLoad::New
(
const polyMesh& mesh,
const word& name,
const bool loadBalancing
)
{
if (loadBalancing && isA<fvMesh>(mesh))
{
return New(refCast<const fvMesh>(mesh), name, loadBalancing);
}
else
{
return optionalCpuLoad::optionalCpuLoad_;
}
}
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
void Foam::cpuLoad::reset()

13
src/finiteVolume/fvMesh/fvMeshDistributors/cpuLoad/cpuLoad.H
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2022 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2022-2024 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -77,6 +77,8 @@ public:
// Selectors
//- Construct from fvMesh if loadBalancing is true
// otherwise return the dummy optionalCpuLoad
static optionalCpuLoad& New
(
const fvMesh& mesh,
@ -84,6 +86,15 @@ public:
const bool loadBalancing
);
//- Construct from polyMesh if it is an fvMesh and loadBalancing is true
// otherwise return the dummy optionalCpuLoad
static optionalCpuLoad& New
(
const polyMesh& mesh,
const word& name,
const bool loadBalancing
);
//- Destructor
virtual ~optionalCpuLoad()

162
src/fvMeshDistributors/loadBalancer/fvMeshDistributorsLoadBalancer.C
View File

@ -85,12 +85,13 @@ bool Foam::fvMeshDistributors::loadBalancer::update()
if
(
Pstream::nProcs() > 1
&& mesh.time().timeIndex() > 1
&& mesh.time().timeIndex() - mesh.time().startTimeIndex() > 1
&& timeIndex_ != mesh.time().timeIndex()
)
{
timeIndex_ = mesh.time().timeIndex();
// Get the CPU time fer this processor which includes waiting time
const scalar timeStepCpuTime = cpuTime_.cpuTimeIncrement();
// CPU loads per cell
@ -107,80 +108,125 @@ bool Foam::fvMeshDistributors::loadBalancer::update()
{
timeIndex_ = mesh.time().timeIndex();
scalar sumCpuLoad = 0;
scalarList procCpuLoads(cpuLoads.size());
label l = 0;
forAllConstIter(HashTable<cpuLoad*>, cpuLoads, iter)
{
sumCpuLoad += sum(iter()->primitiveField());
procCpuLoads[l++] = sum(iter()->primitiveField());
}
const scalar cellCFDCpuTime = returnReduce
(
(timeStepCpuTime - sumCpuLoad)/mesh.nCells(),
minOp<scalar>()
);
List<scalarList> allProcCpuLoads(Pstream::nProcs());
allProcCpuLoads[Pstream::myProcNo()] = procCpuLoads;
Pstream::gatherList(allProcCpuLoads);
Pstream::scatterList(allProcCpuLoads);
// Total CPU time for this processor
const scalar processorCpuTime =
mesh.nCells()*cellCFDCpuTime + sumCpuLoad;
scalarList sumProcCpuLoads(procCpuLoads.size(), scalar(0));
scalarList maxProcCpuLoads(procCpuLoads.size(), scalar(0));
forAll(maxProcCpuLoads, l)
{
forAll(allProcCpuLoads, proci)
{
sumProcCpuLoads[l] += allProcCpuLoads[proci][l];
maxProcCpuLoads[l] =
max(maxProcCpuLoads[l], allProcCpuLoads[proci][l]);
}
}
// Sum over loads of the maximum load CPU time per processor
const scalar sumMaxProcCpuLoad(sum(maxProcCpuLoads));
// Maximum number of cells per processor
const label maxNcells = returnReduce(mesh.nCells(), maxOp<label>());
// Maximum processor CPU time spent doing basic CFD
const scalar maxBaseCpuTime =
returnReduce(timeStepCpuTime, maxOp<scalar>())
- sumMaxProcCpuLoad;
const scalar cellBaseCpuTime = maxBaseCpuTime/maxNcells;
// Processor CPU time spent doing basic CFD, not waiting
const scalar baseCpuTime = mesh.nCells()*cellBaseCpuTime;
// Maximum total CPU time
const scalar maxProcCpuTime = maxBaseCpuTime + sumMaxProcCpuLoad;
// Total CPU time for this processor not waiting
const scalar procCpuTime = baseCpuTime + sum(procCpuLoads);
// Average processor CPU time
const scalar averageProcessorCpuTime =
returnReduce(processorCpuTime, sumOp<scalar>())
/Pstream::nProcs();
returnReduce(procCpuTime, sumOp<scalar>())/Pstream::nProcs();
Pout<< "imbalance "
<< " " << sumCpuLoad
<< " " << mesh.nCells()*cellCFDCpuTime
<< " " << processorCpuTime
<< " " << averageProcessorCpuTime << endl;
const scalar imbalance =
(maxProcCpuTime - averageProcessorCpuTime)
/averageProcessorCpuTime;
const scalar imbalance = returnReduce
(
mag(1 - processorCpuTime/averageProcessorCpuTime),
maxOp<scalar>()
);
Info<< nl << type() << nl;
scalarField weights;
if (multiConstraint_)
l = 0;
forAllConstIter(HashTable<cpuLoad*>, cpuLoads, iter)
{
const int nWeights = cpuLoads.size() + 1;
weights.setSize(nWeights*mesh.nCells());
for (label i=0; i<mesh.nCells(); i++)
{
weights[nWeights*i] = cellCFDCpuTime;
}
label loadi = 1;
forAllConstIter(HashTable<cpuLoad*>, cpuLoads, iter)
{
const scalarField& cpuLoadField = iter()->primitiveField();
forAll(cpuLoadField, i)
{
weights[nWeights*i + loadi] = cpuLoadField[i];
}
loadi++;
}
Info<< " Imbalance of load " << iter()->name() << ": "
<< (
maxProcCpuLoads[l]
- sumProcCpuLoads[l]/Pstream::nProcs()
)/averageProcessorCpuTime
<< endl;
}
else
{
weights.setSize(mesh.nCells(), cellCFDCpuTime);
forAllConstIter(HashTable<cpuLoad*>, cpuLoads, iter)
{
weights += iter()->primitiveField();
}
}
Info<< " Imbalance of base load " << ": "
<< (
maxBaseCpuTime
- mesh.globalData().nTotalCells()*cellBaseCpuTime
/Pstream::nProcs()
)/averageProcessorCpuTime
<< endl;
Info<< " Total imbalance " << imbalance << endl;
if (imbalance > maxImbalance_)
{
Info<< "Redistributing mesh with imbalance "
<< imbalance << endl;
Info<< " Redistributing mesh" << endl;
scalarField weights;
if (multiConstraint_)
{
const label nWeights = cpuLoads.size() + 1;
weights.setSize(nWeights*mesh.nCells());
for (label i=0; i<mesh.nCells(); i++)
{
weights[nWeights*i] = cellBaseCpuTime;
}
label l = 1;
forAllConstIter(HashTable<cpuLoad*>, cpuLoads, iter)
{
const scalarField& cpuLoadField =
iter()->primitiveField();
forAll(cpuLoadField, i)
{
weights[nWeights*i + l] = cpuLoadField[i];
}
l++;
}
}
else
{
weights.setSize(mesh.nCells(), cellBaseCpuTime);
forAllConstIter(HashTable<cpuLoad*>, cpuLoads, iter)
{
weights += iter()->primitiveField();
}
}
// Create new decomposition distribution
const labelList distribution
@ -191,6 +237,8 @@ bool Foam::fvMeshDistributors::loadBalancer::update()
distribute(distribution);
redistributed = true;
Info<< endl;
}
}

20
src/lagrangian/basic/Cloud/Cloud.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2011-2023 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2024 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -35,6 +35,7 @@ License
#include "OFstream.H"
#include "wallPolyPatch.H"
#include "nonConformalCyclicPolyPatch.H"
#include "cpuLoad.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
@ -301,6 +302,11 @@ void Foam::Cloud<ParticleType>::move
List<IDLList<ParticleType>> sendParticles(Pstream::nProcs());
List<DynamicList<label>> sendPatchIndices(Pstream::nProcs());
optionalCpuLoad& cloudCpuTime
(
optionalCpuLoad::New(pMesh_, name() + ":cpuLoad", cloud.cpuLoad())
);
// While there are particles to transfer
while (true)
{
@ -311,6 +317,11 @@ void Foam::Cloud<ParticleType>::move
sendPatchIndices[proci].clear();
}
if (cloud.cpuLoad())
{
cloudCpuTime.reset();
}
// Loop over all particles
forAllIter(typename Cloud<ParticleType>, *this, pIter)
{
@ -319,6 +330,11 @@ void Foam::Cloud<ParticleType>::move
// Move the particle
const bool keepParticle = p.move(cloud, td);
if (cloud.cpuLoad())
{
cloudCpuTime.cpuTimeIncrement(p.cell());
}
// If the particle is to be kept
if (keepParticle)
{
@ -724,7 +740,7 @@ void Foam::Cloud<ParticleType>::distribute(const polyDistributionMap& map)
if (lostCount != 0)
{
WarningInFunction
<< "Mesh-to-mesh mapping of cloud " << this->name()
<< "Distribution of cloud " << this->name()
<< " lost " << lostCount << " particles" << endl;
}
}

9
src/lagrangian/basic/Cloud/Cloud.H
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2011-2023 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2024 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -194,6 +194,13 @@ public:
return IDLList<ParticleType>::size();
};
//- Return true to cache per-cell CPU load
// Overridden in derived clouds, defaults to false
bool cpuLoad() const
{
return false;
}
// Iterators

3
src/lagrangian/parcel/clouds/Templates/MomentumCloud/MomentumCloud.C
View File

@ -300,6 +300,7 @@ Foam::MomentumCloud<CloudType>::MomentumCloud
(
particleProperties_.subOrEmptyDict("subModels", true)
),
cpuLoad_(particleProperties_.lookupOrDefault("cpuLoad", false)),
rndGen_(0),
cellOccupancyPtr_(),
cellLengthScale_(mag(cbrt(this->mesh().V()))),
@ -411,6 +412,7 @@ Foam::MomentumCloud<CloudType>::MomentumCloud
solution_(c.solution_),
constProps_(c.constProps_),
subModelProperties_(c.subModelProperties_),
cpuLoad_(c.cpuLoad_),
rndGen_(c.rndGen_),
cellOccupancyPtr_(nullptr),
cellLengthScale_(c.cellLengthScale_),
@ -500,6 +502,7 @@ Foam::MomentumCloud<CloudType>::MomentumCloud
solution_(mesh),
constProps_(),
subModelProperties_(dictionary::null),
cpuLoad_(c.cpuLoad_),
rndGen_(0),
cellOccupancyPtr_(nullptr),
cellLengthScale_(c.cellLengthScale_),

9
src/lagrangian/parcel/clouds/Templates/MomentumCloud/MomentumCloud.H
View File

@ -155,6 +155,9 @@ protected:
//- Sub-models dictionary
const dictionary subModelProperties_;
//- Switch to enable per-cell CPU load caching for load-balancing
bool cpuLoad_;
//- Random number generator - used by some injection routines
mutable randomGenerator rndGen_;
@ -531,6 +534,12 @@ public:
// Cloud evolution functions
//- Return true to cache per-cell CPU load
bool cpuLoad() const
{
return cpuLoad_;
}
//- Set parcel thermo properties
void setParcelThermoProperties(parcelType& parcel);

7
src/lagrangian/parcel/submodels/Momentum/PatchInteractionModel/LocalInteraction/LocalInteraction.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2011-2022 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2024 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -318,6 +318,11 @@ bool Foam::LocalInteraction<CloudType>::correct
{
const label patchi = pp.index();
if (isA<processorPolyPatch>(pp))
{
return false;
}
switch (patchInteractionTypes_[patchi])
{
case PatchInteractionModel<CloudType>::itNone:

14
src/thermophysicalModels/chemistryModel/chemistryModel/chemistryModel.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2016-2023 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2016-2024 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -38,7 +38,7 @@ Foam::chemistryModel<ThermoType>::chemistryModel
:
odeChemistryModel(thermo),
log_(this->lookupOrDefault("log", false)),
loadBalancing_(this->lookupOrDefault("loadBalancing", false)),
cpuLoad_(this->lookupOrDefault("cpuLoad", false)),
jacobianType_
(
this->found("jacobian")
@ -582,9 +582,9 @@ Foam::scalar Foam::chemistryModel<ThermoType>::solve
const DeltaTType& deltaT
)
{
optionalCpuLoad& chemistryCpuTime
optionalCpuLoad& chemistryCpuLoad
(
optionalCpuLoad::New(this->mesh(), "chemistryCpuTime", loadBalancing_)
optionalCpuLoad::New(this->mesh(), name() + ":cpuLoad", cpuLoad_)
);
// CPU time logging
@ -617,7 +617,7 @@ Foam::scalar Foam::chemistryModel<ThermoType>::solve
scalar deltaTMin = great;
tabulation_.reset();
chemistryCpuTime.reset();
chemistryCpuLoad.reset();
forAll(rho0vf, celli)
{
@ -763,9 +763,9 @@ Foam::scalar Foam::chemistryModel<ThermoType>::solve
RR_[i][celli] = rho0*(Y_[i] - Y0[i])/deltaT[celli];
}
if (loadBalancing_)
if (cpuLoad_)
{
chemistryCpuTime.cpuTimeIncrement(celli);
chemistryCpuLoad.cpuTimeIncrement(celli);
}
}

6
src/thermophysicalModels/chemistryModel/chemistryModel/chemistryModel.H
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Copyright (C) 2016-2023 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2016-2024 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -128,8 +128,8 @@ class chemistryModel
//- Switch to select performance logging
Switch log_;
//- Switch to enable loadBalancing performance logging
Switch loadBalancing_;
//- Switch to enable per-cell CPU load caching for load-balancing
Switch cpuLoad_;
//- Type of the Jacobian to be calculated
const jacobianType jacobianType_;

2
tutorials/incompressibleDenseParticleFluid/cyclone/0/U.air
View File

@ -20,6 +20,8 @@ internalField uniform (0 0 0);
boundaryField
{
#includeEtc "caseDicts/setConstraintTypes"
inlet
{
type fixedValue;

2
tutorials/incompressibleDenseParticleFluid/cyclone/0/k.air
View File

@ -19,6 +19,8 @@ internalField uniform 1;
boundaryField
{
#includeEtc "caseDicts/setConstraintTypes"
inlet
{
type fixedValue;

2
tutorials/incompressibleDenseParticleFluid/cyclone/0/nut.air
View File

@ -19,6 +19,8 @@ internalField uniform 0;
boundaryField
{
#includeEtc "caseDicts/setConstraintTypes"
inlet
{
type calculated;

2
tutorials/incompressibleDenseParticleFluid/cyclone/0/p
View File

@ -19,6 +19,8 @@ internalField uniform 0;
boundaryField
{
#includeEtc "caseDicts/setConstraintTypes"
inlet
{
type fixedFluxPressure;

2
tutorials/incompressibleDenseParticleFluid/cyclone/constant/cloudProperties
View File

@ -16,6 +16,8 @@ FoamFile
type MPPICCloud;
cpuLoad true;
solution
{
coupled true;

27
tutorials/incompressibleDenseParticleFluid/cyclone/constant/dynamicMeshDict Normal file
View File

@ -0,0 +1,27 @@
/*--------------------------------*- C++ -*----------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration | Website: https://openfoam.org
\\ / A nd | Version: dev
\\/ M anipulation |
\*---------------------------------------------------------------------------*/
FoamFile
{
format ascii;
class dictionary;
object dynamicMeshDict;
}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
distributor
{
type loadBalancer;
libs ("libfvMeshDistributors.so");
multiConstraint true;
redistributionInterval 10;
}
// ************************************************************************* //

6
tutorials/incompressibleDenseParticleFluid/cyclone/system/blockMeshDict
View File

@ -58,6 +58,12 @@ boundary
(4 5 6 7)
);
}
internalFaces
{
type internal;
faces ();
}
);
// ************************************************************************* //

2
tutorials/incompressibleDenseParticleFluid/cyclone/system/controlDict
View File

@ -18,7 +18,7 @@ application foamRun;
solver incompressibleDenseParticleFluid;
startFrom latestTime;
startFrom startTime;
startTime 0;

18
tutorials/incompressibleDenseParticleFluid/cyclone/system/decomposeParDict
View File

@ -16,11 +16,25 @@ FoamFile
numberOfSubdomains 12;
method simple;
decomposer simple;
distributor zoltan;
libs ("libzoltanDecomp.so");
simpleCoeffs
{
n (2 2 3);
n (2 2 3);
}
hierarchicalCoeffs
{
n (2 2 3);
order xyz;
}
zoltanCoeffs
{
lb_method rcb;
}
// ************************************************************************* //

2
tutorials/multicomponentFluid/counterFlowFlame2D_GRI/constant/chemistryProperties
View File

@ -21,7 +21,7 @@ chemistryType
chemistry on;
loadBalancing on;
cpuLoad on;
initialChemicalTimeStep 1e-07;

2
tutorials/multicomponentFluid/counterFlowFlame2D_GRI/system/blockMeshDict
View File

@ -69,7 +69,7 @@ boundary
);
}
internal
internalFaces
{
type internal;
faces ();