fvMeshDistributors::cpuLoad: New class to simply caching multiple CPU loads per cell

Used by fvMeshDistributors::loadBalancer to generate the multi-constraint
weights for mesh redistribution.

src/finiteVolume/Make/files

@@ -104,6 +104,7 @@ fvMeshDistributors = fvMesh/fvMeshDistributors
 $(fvMeshDistributors)/fvMeshDistributor/fvMeshDistributor.C
 $(fvMeshDistributors)/fvMeshDistributor/fvMeshDistributorNew.C
 $(fvMeshDistributors)/none/fvMeshDistributorsNone.C
+$(fvMeshDistributors)/cpuLoad/cpuLoad.C
 
 fvMeshMovers = fvMesh/fvMeshMovers
 

src/finiteVolume/fvMesh/fvMeshDistributors/cpuLoad/cpuLoad.C

@@ -0,0 +1,124 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     | Website:  https://openfoam.org
+    \\  /    A nd           | Copyright (C) 2022 OpenFOAM Foundation
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "cpuLoad.H"
+
+// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
+
+namespace Foam
+{
+    defineTypeNameAndDebug(cpuLoad, 0);
+    optionalCpuLoad optionalCpuLoad::optionalCpuLoad_;
+}
+
+
+// * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
+
+Foam::cpuLoad::cpuLoad(const fvMesh& mesh, const word& name)
+:
+    volScalarField::Internal
+    (
+        IOobject
+        (
+            name,
+            mesh.time().timeName(),
+            mesh
+        ),
+        mesh,
+        dimensionedScalar(dimTime, 0)
+    )
+{}
+
+
+// * * * * * * * * * * * * * * * * Destructor  * * * * * * * * * * * * * * * //
+
+Foam::cpuLoad::~cpuLoad()
+{}
+
+
+// * * * * * * * * * * * * * * * * Selectors * * * * * * * * * * * * * * * * //
+
+Foam::optionalCpuLoad& Foam::optionalCpuLoad::New
+(
+    const fvMesh& mesh,
+    const word& name,
+    const bool loadBalancing
+)
+{
+    if (loadBalancing)
+    {
+        if
+        (
+            mesh.thisDb().objectRegistry::template
+            foundObject<cpuLoad>
+            (
+                name
+            )
+        )
+        {
+            return mesh.thisDb().objectRegistry::template
+            lookupObjectRef<cpuLoad>
+            (
+                name
+            );
+        }
+        else
+        {
+            if (cpuLoad::debug)
+            {
+                InfoInFunction
+                    << "constructing " << name
+                    << " for region " << mesh.name() << endl;
+            }
+
+            cpuLoad* cpuLoadPtr(new cpuLoad(mesh, name));
+
+            regIOobject::store(cpuLoadPtr);
+
+            return *cpuLoadPtr;
+        }
+    }
+    else
+    {
+        return optionalCpuLoad::optionalCpuLoad_;
+    }
+}
+
+
+// * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
+
+void Foam::cpuLoad::reset()
+{
+    cpuTime_.cpuTimeIncrement();
+}
+
+
+void Foam::cpuLoad::cpuTimeIncrement(const label celli)
+{
+    operator[](celli) += cpuTime_.cpuTimeIncrement();
+}
+
+
+// ************************************************************************* //

src/finiteVolume/fvMesh/fvMeshDistributors/cpuLoad/cpuLoad.H

@@ -0,0 +1,172 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     | Website:  https://openfoam.org
+    \\  /    A nd           | Copyright (C) 2022 OpenFOAM Foundation
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+Class
+    Foam::cpuLoad
+
+Description
+    Class to maintain a field of the CPU load per cell.
+
+    The optionalCpuLoad is a base-class for cpuLoad to simplify the
+    implementation of optional CPU time caching.  This is achieved via the
+    optionalCpuLoad::New function which returns a optionalCpuLoad with dummy
+    functions if loadBalancing is false otherwise it creates or looks-up and
+    returns a cpuLoad with the given name.
+
+    Used for loadBalancing.
+
+SourceFiles
+    cpuLoad.C
+
+\*---------------------------------------------------------------------------*/
+
+#ifndef cpuLoad_H
+#define cpuLoad_H
+
+#include "volFields.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+/*---------------------------------------------------------------------------*\
+                       Class optionalCpuLoad Declaration
+\*---------------------------------------------------------------------------*/
+
+class optionalCpuLoad
+{
+    // Private Member Data
+
+        //- Dummy optionalCpuLoad
+        //  returned by optionalCpuLoad::New if loadBalancing = false
+        static optionalCpuLoad optionalCpuLoad_;
+
+
+public:
+
+    // Constructors
+
+        //- Construct from mesh, name and switch
+        optionalCpuLoad()
+        {}
+
+        //- Disallow default bitwise copy construction
+        optionalCpuLoad(const optionalCpuLoad&) = delete;
+
+
+    // Selectors
+
+        static optionalCpuLoad& New
+        (
+            const fvMesh& mesh,
+            const word& name,
+            const bool loadBalancing
+        );
+
+
+    //- Destructor
+    virtual ~optionalCpuLoad()
+    {}
+
+
+    // Member Functions
+
+        //- Dummy reset function
+        virtual void reset()
+        {}
+
+        //- Dummy cpuTimeIncrement function
+        virtual void cpuTimeIncrement(const label celli)
+        {}
+
+
+    // Member Operators
+
+        //- Disallow default bitwise assignment
+        void operator=(const optionalCpuLoad&) = delete;
+};
+
+
+/*---------------------------------------------------------------------------*\
+                          Class cpuLoad Declaration
+\*---------------------------------------------------------------------------*/
+
+class cpuLoad
+:
+    public optionalCpuLoad,
+    public volScalarField::Internal
+{
+    // Private Data
+
+        cpuTime cpuTime_;
+
+
+public:
+
+    // Declare name of the class and its debug switch
+    TypeName("cpuLoad");
+
+
+    // Constructors
+
+        //- Construct from mesh, name and switch
+        cpuLoad
+        (
+            const fvMesh& mesh,
+            const word& name
+        );
+
+        //- Disallow default bitwise copy construction
+        cpuLoad(const cpuLoad&) = delete;
+
+
+    //- Destructor
+    virtual ~cpuLoad();
+
+
+    // Member Functions
+
+        //- Reset the CPU time
+        virtual void reset();
+
+        //- Cache the CPU time increment for celli
+        virtual void cpuTimeIncrement(const label celli);
+
+
+    // Member Operators
+
+        //- Disallow default bitwise assignment
+        void operator=(const cpuLoad&) = delete;
+};
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace Foam
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+#endif
+
+// ************************************************************************* //

src/fvMeshDistributors/loadBalancer/fvMeshDistributorsLoadBalancer.C

@@ -25,7 +25,7 @@ License
 
 #include "fvMeshDistributorsLoadBalancer.H"
 #include "decompositionMethod.H"
-#include "volFields.H"
+#include "cpuLoad.H"
 #include "addToRunTimeSelectionTable.H"
 
 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
@@ -93,20 +93,26 @@ bool Foam::fvMeshDistributors::loadBalancer::update()
 
         const scalar timeStepCpuTime = cpuTime_.cpuTimeIncrement();
 
-        // Chemistry CPU load per cell
+        // CPU loads per cell
-        volScalarField::Internal& chemistryCpuTimeReg =
+        HashTable<cpuLoad*> cpuLoads(this->mesh().lookupClass<cpuLoad>());
-            mesh.lookupObjectRef<volScalarField::Internal>
-            (
-                "chemistryCpuTime"
-            );
 
-        const scalarField& chemistryCpuTime = chemistryCpuTimeReg.field();
+        if (!cpuLoads.size())
+        {
+            FatalErrorInFunction
+                << "No CPU loads have been allocated"
+                << exit(FatalError);
+        }
 
         if (mesh.time().timeIndex() % redistributionInterval_ == 0)
         {
             timeIndex_ = mesh.time().timeIndex();
 
-            const scalar sumCpuLoad(sum(chemistryCpuTime));
+            scalar sumCpuLoad = 0;
+
+            forAllConstIter(HashTable<cpuLoad*>, cpuLoads, iter)
+            {
+                sumCpuLoad += sum(iter()->field());
+            }
 
             const scalar cellCFDCpuTime = returnReduce
             (
@@ -139,19 +145,36 @@ bool Foam::fvMeshDistributors::loadBalancer::update()
 
             if (multiConstraint_)
             {
-                const int nWeights = 2;
+                const int nWeights = cpuLoads.size() + 1;
 
                 weights.setSize(nWeights*mesh.nCells());
 
-                forAll(chemistryCpuTime, i)
+                for (label i=0; i<mesh.nCells(); i++)
                 {
                     weights[nWeights*i] = cellCFDCpuTime;
-                    weights[nWeights*i + 1] = chemistryCpuTime[i];
+                }
+
+                label loadi = 1;
+                forAllConstIter(HashTable<cpuLoad*>, cpuLoads, iter)
+                {
+                    const scalarField& cpuLoadField = iter()->field();
+
+                    forAll(cpuLoadField, i)
+                    {
+                        weights[nWeights*i + loadi] = cpuLoadField[i];
+                    }
+
+                    loadi++;
                 }
             }
             else
             {
-                weights = chemistryCpuTime + cellCFDCpuTime;
+                weights.setSize(mesh.nCells(), cellCFDCpuTime);
+
+                forAllConstIter(HashTable<cpuLoad*>, cpuLoads, iter)
+                {
+                    weights += iter()->field();
+                }
             }
 
             if (imbalance > maxImbalance_)
@@ -171,7 +194,10 @@ bool Foam::fvMeshDistributors::loadBalancer::update()
             }
         }
 
-        chemistryCpuTimeReg.checkOut();
+        forAllIter(HashTable<cpuLoad*>, cpuLoads, iter)
+        {
+            iter()->checkOut();
+        }
     }
 
     return redistributed;

src/thermophysicalModels/chemistryModel/chemistryModel/chemistryModel.C

@@ -26,7 +26,7 @@ License
 #include "chemistryModel.H"
 #include "UniformField.H"
 #include "localEulerDdtScheme.H"
-#include "cpuTime.H"
+#include "cpuLoad.H"
 
 // * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
 
@@ -710,41 +710,16 @@ Foam::scalar Foam::chemistryModel<ThermoType>::solve
     const DeltaTType& deltaT
 )
 {
-    if (loadBalancing_)
-    {
-        if
-        (
-            !this->mesh().objectRegistry::template
-            foundObject<volScalarField::Internal>("chemistryCpuTime")
-        )
-        {
-            regIOobject::store
-            (
-                volScalarField::Internal::New
-                (
-                    "chemistryCpuTime",
-                    this->mesh(),
-                    dimensionedScalar(dimTime, 0)
-                ).ptr()
-            );
-        }
-    }
-
-    volScalarField::Internal& chemistryCpuTime =
-        loadBalancing_
-      ? this->mesh().objectRegistry::template
-        lookupObjectRef<volScalarField::Internal>
-        (
-            "chemistryCpuTime"
-        )
-    : const_cast<volScalarField::Internal&>(volScalarField::Internal::null());
-
     tabulation_.reset();
 
     const basicSpecieMixture& composition = this->thermo().composition();
 
+    optionalCpuLoad& chemistryCpuTime
+    (
+        optionalCpuLoad::New(this->mesh(), "chemistryCpuTime", loadBalancing_)
+    );
+
     // CPU time analysis
-    cpuTime cpuTime_;
     cpuTime solveCpuTime_;
     scalar totalSolveCpuTime_ = 0;
 
@@ -773,6 +748,8 @@ Foam::scalar Foam::chemistryModel<ThermoType>::solve
     scalarField phiq(nEqns() + 1);
     scalarField Rphiq(nEqns() + 1);
 
+    chemistryCpuTime.reset();
+
     forAll(rho0vf, celli)
     {
         const scalar rho = rhovf[celli];
@@ -913,7 +890,7 @@ Foam::scalar Foam::chemistryModel<ThermoType>::solve
 
         if (loadBalancing_)
         {
-            chemistryCpuTime[celli] += cpuTime_.cpuTimeIncrement();
+            chemistryCpuTime.cpuTimeIncrement(celli);
         }
     }