Merge branch 'feature-parProfiling' into 'develop'

ENH: parProfiling: profile linear solver only

See merge request Development/openfoam!603

src/OpenFOAM/matrices/lduMatrix/lduMatrix/lduMatrix.H

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2017 OpenFOAM Foundation
-    Copyright (C) 2016-2022 OpenCFD Ltd.
+    Copyright (C) 2016-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -229,7 +229,19 @@ public:
 
         // Selectors
 
-            //- Return a new solver
+            //- Return a new solver of given type
+            static autoPtr<solver> New
+            (
+                const word& solverName,
+                const word& fieldName,
+                const lduMatrix& matrix,
+                const FieldField<Field, scalar>& interfaceBouCoeffs,
+                const FieldField<Field, scalar>& interfaceIntCoeffs,
+                const lduInterfaceFieldPtrsList& interfaces,
+                const dictionary& solverControls
+            );
+
+            //- Return a new solver given dictionary
             static autoPtr<solver> New
             (
                 const word& fieldName,

src/OpenFOAM/matrices/lduMatrix/lduMatrix/lduMatrixSolver.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2017 OpenFOAM Foundation
-    Copyright (C) 2016-2022 OpenCFD Ltd.
+    Copyright (C) 2016-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -39,10 +39,11 @@ namespace Foam
 }
 
 
-// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+// * * * * * * * * * * * * * * * * Selectors * * * * * * * * * * * * * * * * //
 
 Foam::autoPtr<Foam::lduMatrix::solver> Foam::lduMatrix::solver::New
 (
+    const word& solverName,
     const word& fieldName,
     const lduMatrix& matrix,
     const FieldField<Field, scalar>& interfaceBouCoeffs,
@@ -51,8 +52,6 @@ Foam::autoPtr<Foam::lduMatrix::solver> Foam::lduMatrix::solver::New
     const dictionary& solverControls
 )
 {
-    const word name(solverControls.get<word>("solver"));
-
     if (matrix.diagonal())
     {
         return autoPtr<lduMatrix::solver>
@@ -70,7 +69,7 @@ Foam::autoPtr<Foam::lduMatrix::solver> Foam::lduMatrix::solver::New
     }
     else if (matrix.symmetric())
     {
-        auto* ctorPtr = symMatrixConstructorTable(name);
+        auto* ctorPtr = symMatrixConstructorTable(solverName);
 
         if (!ctorPtr)
         {
@@ -78,7 +77,7 @@ Foam::autoPtr<Foam::lduMatrix::solver> Foam::lduMatrix::solver::New
             (
                 solverControls,
                 "symmetric matrix solver",
-                name,
+                solverName,
                 *symMatrixConstructorTablePtr_
             ) << exit(FatalIOError);
         }
@@ -98,7 +97,7 @@ Foam::autoPtr<Foam::lduMatrix::solver> Foam::lduMatrix::solver::New
     }
     else if (matrix.asymmetric())
     {
-        auto* ctorPtr = asymMatrixConstructorTable(name);
+        auto* ctorPtr = asymMatrixConstructorTable(solverName);
 
         if (!ctorPtr)
         {
@@ -106,7 +105,7 @@ Foam::autoPtr<Foam::lduMatrix::solver> Foam::lduMatrix::solver::New
             (
                 solverControls,
                 "asymmetric matrix solver",
-                name,
+                solverName,
                 *asymMatrixConstructorTablePtr_
             ) << exit(FatalIOError);
         }
@@ -134,6 +133,29 @@ Foam::autoPtr<Foam::lduMatrix::solver> Foam::lduMatrix::solver::New
 }
 
 
+Foam::autoPtr<Foam::lduMatrix::solver> Foam::lduMatrix::solver::New
+(
+    const word& fieldName,
+    const lduMatrix& matrix,
+    const FieldField<Field, scalar>& interfaceBouCoeffs,
+    const FieldField<Field, scalar>& interfaceIntCoeffs,
+    const lduInterfaceFieldPtrsList& interfaces,
+    const dictionary& solverControls
+)
+{
+    return New
+    (
+        solverControls.get<word>("solver"),
+        fieldName,
+        matrix,
+        interfaceBouCoeffs,
+        interfaceIntCoeffs,
+        interfaces,
+        solverControls
+    );
+}
+
+
 // * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
 
 Foam::lduMatrix::solver::solver

src/OpenFOAM/matrices/lduMatrix/solvers/PCG/PCG.C

@@ -210,7 +210,6 @@ Foam::solverPerformance Foam::PCG::scalarSolve
 }
 
 
-
 Foam::solverPerformance Foam::PCG::solve
 (
     scalarField& psi_s,

src/functionObjects/utilities/Make/files

@@ -15,6 +15,7 @@ multiRegion/multiRegion.C
 removeRegisteredObject/removeRegisteredObject.C
 
 parProfiling/parProfiling.C
+parProfiling/parProfilingSolver.C
 
 solverInfo/solverInfo.C
 timeInfo/timeInfo.C

src/functionObjects/utilities/parProfiling/parProfilingSolver.C

@@ -0,0 +1,127 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | www.openfoam.com
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+    Copyright (C) 2023 OpenCFD Ltd.
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "lduMatrix.H"
+#include "parProfilingSolver.H"
+#include "profilingPstream.H"
+#include "addToRunTimeSelectionTable.H"
+
+// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
+
+namespace Foam
+{
+    defineTypeNameAndDebug(parProfilingSolver, 0);
+
+    typedef lduMatrix::solver baseType;
+
+    addNamedToRunTimeSelectionTable
+    (
+        baseType,
+        parProfilingSolver,
+        symMatrix,
+        parProfiling
+    );
+
+    addNamedToRunTimeSelectionTable
+    (
+        baseType,
+        parProfilingSolver,
+        asymMatrix,
+        parProfiling
+    );
+}
+
+
+// Has been initialised
+static bool initialised_(false);
+
+
+// * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
+
+Foam::parProfilingSolver::parProfilingSolver
+(
+    const word& fieldName,
+    const lduMatrix& matrix,
+    const FieldField<Field, scalar>& interfaceBouCoeffs,
+    const FieldField<Field, scalar>& interfaceIntCoeffs,
+    const lduInterfaceFieldPtrsList& interfaces,
+    const dictionary& solverControls
+)
+:
+    lduMatrix::solver
+    (
+        fieldName,
+        matrix,
+        interfaceBouCoeffs,
+        interfaceIntCoeffs,
+        interfaces,
+        solverControls
+    )
+{
+    if (!initialised_)
+    {
+        initialised_ = true;
+        profilingPstream::reset();
+        profilingPstream::suspend();
+    }
+
+    const word baseSolver(solverControls.get<word>("baseSolver"));
+
+    solvePtr_.reset
+    (
+        lduMatrix::solver::New
+        (
+            baseSolver,
+            fieldName,
+            matrix,
+            interfaceBouCoeffs,
+            interfaceIntCoeffs,
+            interfaces,
+            solverControls
+        )
+    );
+}
+
+
+// * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
+
+Foam::solverPerformance Foam::parProfilingSolver::solve
+(
+    scalarField& psi_s,
+    const scalarField& source,
+    const direction cmpt
+) const
+{
+    profilingPstream::enable();
+    Foam::solverPerformance perf(solvePtr_->solve(psi_s, source, cmpt));
+    profilingPstream::suspend();
+
+    return perf;
+}
+
+
+// ************************************************************************* //

src/functionObjects/utilities/parProfiling/parProfilingSolver.H

@@ -0,0 +1,132 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | www.openfoam.com
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+    Copyright (C) 2023 OpenCFD Ltd.
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+Class
+    Foam::parProfilingSolver
+
+Description
+    Wrapper to switch on parProfiling around a linear solver.
+
+    Used in combination with parProfiling functionObject.
+
+Usage
+    Example of linear solver specification in fvSolution:
+    \verbatim
+    solvers
+    {
+        p
+        {
+            solver          parProfiling;
+            // Actual solver to use
+            baseSolver      PCG;
+            preconditioner  DIC;
+            ..
+        }
+    }
+    \endverbatim
+
+SourceFiles
+    parProfiling.C
+
+\*---------------------------------------------------------------------------*/
+
+#ifndef Foam_parProfilingSolver_H
+#define Foam_parProfilingSolver_H
+
+#include "lduMatrix.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+/*---------------------------------------------------------------------------*\
+                     Class parProfilingSolver Declaration
+\*---------------------------------------------------------------------------*/
+
+class parProfilingSolver
+:
+    public lduMatrix::solver
+{
+    // Private Data
+
+        //- Underlying solver
+        autoPtr<lduMatrix::solver> solvePtr_;
+
+
+    // Private Member Functions
+
+        //- No copy construct
+        parProfilingSolver(const parProfilingSolver&) = delete;
+
+        //- No copy assignment
+        void operator=(const parProfilingSolver&) = delete;
+
+
+public:
+
+    //- Runtime type information
+    TypeName("parProfilingSolver");
+
+
+    // Constructors
+
+        //- Construct from matrix components and solver controls
+        parProfilingSolver
+        (
+            const word& fieldName,
+            const lduMatrix& matrix,
+            const FieldField<Field, scalar>& interfaceBouCoeffs,
+            const FieldField<Field, scalar>& interfaceIntCoeffs,
+            const lduInterfaceFieldPtrsList& interfaces,
+            const dictionary& solverControls
+        );
+
+
+    //- Destructor
+    virtual ~parProfilingSolver() = default;
+
+
+    // Member Functions
+
+        //- Solve the matrix with forwarding to the base solver
+        virtual solverPerformance solve
+        (
+            scalarField& psi,
+            const scalarField& source,
+            const direction cmpt=0
+        ) const;
+};
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace Foam
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+#endif
+
+// ************************************************************************* //

tutorials/IO/cavity_parProfiling/0/U

@@ -0,0 +1,41 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2306                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version     2.0;
+    format      ascii;
+    class       volVectorField;
+    object      U;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+dimensions      [0 1 -1 0 0 0 0];
+
+internalField   uniform (0 0 0);
+
+boundaryField
+{
+    movingWall
+    {
+        type            fixedValue;
+        value           uniform (1 0 0);
+    }
+
+    fixedWalls
+    {
+        type            noSlip;
+    }
+
+    frontAndBack
+    {
+        type            empty;
+    }
+}
+
+
+// ************************************************************************* //

tutorials/IO/cavity_parProfiling/0/p

@@ -0,0 +1,40 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2306                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version     2.0;
+    format      ascii;
+    class       volScalarField;
+    object      p;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+dimensions      [0 2 -2 0 0 0 0];
+
+internalField   uniform 0;
+
+boundaryField
+{
+    movingWall
+    {
+        type            zeroGradient;
+    }
+
+    fixedWalls
+    {
+        type            zeroGradient;
+    }
+
+    frontAndBack
+    {
+        type            empty;
+    }
+}
+
+
+// ************************************************************************* //

tutorials/IO/cavity_parProfiling/Allclean

@@ -0,0 +1,10 @@
+#!/bin/sh
+cd "${0%/*}" || exit                                # Run from this directory
+. ${WM_PROJECT_DIR:?}/bin/tools/CleanFunctions      # Tutorial clean functions
+#------------------------------------------------------------------------------
+
+cleanCase
+
+rm -f system/fvSolution
+
+#------------------------------------------------------------------------------

tutorials/IO/cavity_parProfiling/Allrun

@@ -0,0 +1,27 @@
+#!/bin/sh
+cd "${0%/*}" || exit                                # Run from this directory
+. ${WM_PROJECT_DIR:?}/bin/tools/RunFunctions        # Tutorial run functions
+#------------------------------------------------------------------------------
+
+cp -f system/fvSolution.template system/fvSolution
+
+runApplication blockMesh
+
+runApplication decomposePar
+
+#- Normal solver. Shows that sleep time is not included in profiling
+
+solver="PCG"
+foamDictionary -entry solvers/p/solver -set "$solver" system/fvSolution
+foamDictionary -entry solvers/pFinal/solver -set "$solver" system/fvSolution
+
+runParallel -s "$solver" $(getApplication)
+
+#- Run again with profiling
+solver="parProfiling"
+foamDictionary -entry solvers/p/solver -set "$solver" system/fvSolution
+foamDictionary -entry solvers/pFinal/solver -set "$solver" system/fvSolution
+
+runParallel -s "$solver" $(getApplication)
+
+#------------------------------------------------------------------------------

tutorials/IO/cavity_parProfiling/README.txt

@@ -0,0 +1,3 @@
+Testcase for parProfiling wrapper linear solver.
+
+It only activates parProfiling within the linear solver.

tutorials/IO/cavity_parProfiling/constant/transportProperties

@@ -0,0 +1,20 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2306                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version     2.0;
+    format      ascii;
+    class       dictionary;
+    object      transportProperties;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+nu              0.01;
+
+
+// ************************************************************************* //

tutorials/IO/cavity_parProfiling/system/blockMeshDict

@@ -0,0 +1,72 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2306                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version     2.0;
+    format      ascii;
+    class       dictionary;
+    object      blockMeshDict;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+scale   0.1;
+
+vertices
+(
+    (0 0 0)
+    (1 0 0)
+    (1 1 0)
+    (0 1 0)
+    (0 0 0.1)
+    (1 0 0.1)
+    (1 1 0.1)
+    (0 1 0.1)
+);
+
+blocks
+(
+    hex (0 1 2 3 4 5 6 7) (20 20 1) simpleGrading (1 1 1)
+);
+
+edges
+(
+);
+
+boundary
+(
+    movingWall
+    {
+        type wall;
+        faces
+        (
+            (3 7 6 2)
+        );
+    }
+    fixedWalls
+    {
+        type wall;
+        faces
+        (
+            (0 4 7 3)
+            (2 6 5 1)
+            (1 5 4 0)
+        );
+    }
+    frontAndBack
+    {
+        type empty;
+        faces
+        (
+            (0 3 2 1)
+            (4 5 6 7)
+        );
+    }
+);
+
+
+// ************************************************************************* //

tutorials/IO/cavity_parProfiling/system/controlDict

@@ -0,0 +1,70 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2306                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version     2.0;
+    format      ascii;
+    class       dictionary;
+    object      controlDict;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+// Can force loading of solver wrapper...
+libs            (utilityFunctionObjects);
+
+application     icoFoam;
+
+startFrom       startTime;
+
+startTime       0;
+
+stopAt          endTime;
+
+endTime         0.1;
+
+deltaT          0.005;
+
+writeControl    timeStep;
+
+writeInterval   20;
+
+purgeWrite      0;
+
+writeFormat     ascii;
+
+writePrecision  6;
+
+writeCompression off;
+
+timeFormat      general;
+
+timePrecision   6;
+
+runTimeModifiable true;
+
+functions
+{
+    // Add some dummy time - shouldn't be seen in profiling
+    sleep
+    {
+        type    coded;
+        libs    (utilityFunctionObjects);
+        name    sleep;
+
+        codeExecute #{ sleep(1); #};
+    }
+
+    // Run parProfiling
+    profiling
+    {
+        #includeEtc "caseDicts/profiling/parallel.cfg"
+        detail  2;
+    }
+}
+
+// ************************************************************************* //

tutorials/IO/cavity_parProfiling/system/decomposeParDict

@@ -0,0 +1,27 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2306                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version     2.0;
+    format      ascii;
+    class       dictionary;
+    object      decomposeParDict;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+numberOfSubdomains  2;
+
+method  hierarchical;
+
+coeffs
+{
+    n   (2 1 1);
+}
+
+
+// ************************************************************************* //

tutorials/IO/cavity_parProfiling/system/fvSchemes

@@ -0,0 +1,50 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2306                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version     2.0;
+    format      ascii;
+    class       dictionary;
+    object      fvSchemes;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+ddtSchemes
+{
+    default         Euler;
+}
+
+gradSchemes
+{
+    default         Gauss linear;
+    grad(p)         Gauss linear;
+}
+
+divSchemes
+{
+    default         none;
+    div(phi,U)      Gauss linear;
+}
+
+laplacianSchemes
+{
+    default         Gauss linear orthogonal;
+}
+
+interpolationSchemes
+{
+    default         linear;
+}
+
+snGradSchemes
+{
+    default         orthogonal;
+}
+
+
+// ************************************************************************* //

tutorials/IO/cavity_parProfiling/system/fvSolution.template

@@ -0,0 +1,53 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2306                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version         2;
+    format          ascii;
+    class           dictionary;
+    object          fvSolution;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+solvers
+{
+    p
+    {
+        solver          parProfiling;
+        baseSolver      PCG;
+        preconditioner  DIC;
+        tolerance       1e-06;
+        relTol          0.05;
+    }
+    pFinal
+    {
+        solver          parProfiling;
+        baseSolver      PCG;
+        preconditioner  DIC;
+        tolerance       1e-06;
+        relTol          0;
+    }
+    U
+    {
+        solver          smoothSolver;
+        smoother        symGaussSeidel;
+        tolerance       1e-05;
+        relTol          0;
+    }
+}
+
+PISO
+{
+    nCorrectors     2;
+    nNonOrthogonalCorrectors 0;
+    pRefCell        0;
+    pRefValue       0;
+}
+
+
+// ************************************************************************* //