ENH: lduMatrix: new matrix solvers: PPCG,PPCR

PPCG is pipelined version of PCG, PPCR is conjugate
residual version.

src/OpenFOAM/Make/files

@@ -383,6 +383,8 @@ $(lduMatrix)/solvers/smoothSolver/smoothSolver.C
 $(lduMatrix)/solvers/PCG/PCG.C
 $(lduMatrix)/solvers/PBiCG/PBiCG.C
 $(lduMatrix)/solvers/PBiCGStab/PBiCGStab.C
+$(lduMatrix)/solvers/PPCG/PPCG.C
+$(lduMatrix)/solvers/PPCR/PPCR.C
 
 $(lduMatrix)/smoothers/GaussSeidel/GaussSeidelSmoother.C
 $(lduMatrix)/smoothers/symGaussSeidel/symGaussSeidelSmoother.C

src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H

@@ -156,6 +156,21 @@ void reduce
     NotImplemented;
 }
 
+// Non-blocking version of reduce. Sets request.
+template<class T, class BinaryOp>
+void reduce
+(
+    T Value[],
+    const int size,
+    const BinaryOp& bop,
+    const int tag,
+    const label comm,
+    label& request
+)
+{
+    NotImplemented;
+}
+
 
 // Insist there are specialisations for the common reductions of scalar(s)
 void reduce
@@ -199,6 +214,16 @@ void reduce
     label& request
 );
 
+void reduce
+(
+    scalar Value[],
+    const int size,
+    const sumOp<scalar>& bop,
+    const int tag,
+    const label comm,
+    label& request
+);
+
 
 #if defined(WM_SPDP)
 void reduce
@@ -241,6 +266,16 @@ void reduce
     const label comm,
     label& request
 );
+
+void reduce
+(
+    solveScalar Value[],
+    const int size,
+    const sumOp<solveScalar>& bop,
+    const int tag,
+    const label comm,
+    label& request
+);
 #endif
 
 

src/OpenFOAM/matrices/lduMatrix/lduMatrix/lduMatrix.H

@@ -708,7 +708,8 @@ public:
                 const lduInterfaceFieldPtrsList& interfaces,
                 const solveScalarField& psiif,
                 solveScalarField& result,
-                const direction cmpt
+                const direction cmpt,
+                const label startRequest // starting request (for non-blocking)
             ) const;
 
             //- Set the residual field using an IOField on the object registry

src/OpenFOAM/matrices/lduMatrix/lduMatrix/lduMatrixATmul.C

@@ -56,6 +56,8 @@ void Foam::lduMatrix::Amul
     const scalar* const __restrict__ upperPtr = upper().begin();
     const scalar* const __restrict__ lowerPtr = lower().begin();
 
+    const label startRequest = Pstream::nRequests();
+
     // Initialise the update of interfaced interfaces
     initMatrixInterfaces
     (
@@ -90,7 +92,8 @@ void Foam::lduMatrix::Amul
         interfaces,
         psi,
         Apsi,
-        cmpt
+        cmpt,
+        startRequest
     );
 
     tpsi.clear();
@@ -119,6 +122,8 @@ void Foam::lduMatrix::Tmul
     const scalar* const __restrict__ lowerPtr = lower().begin();
     const scalar* const __restrict__ upperPtr = upper().begin();
 
+    const label startRequest = Pstream::nRequests();
+
     // Initialise the update of interfaced interfaces
     initMatrixInterfaces
     (
@@ -151,7 +156,8 @@ void Foam::lduMatrix::Tmul
         interfaces,
         psi,
         Tpsi,
-        cmpt
+        cmpt,
+        startRequest
     );
 
     tpsi.clear();
@@ -240,6 +246,8 @@ void Foam::lduMatrix::residual
     // To compensate for this, it is necessary to turn the
     // sign of the contribution.
 
+    const label startRequest = Pstream::nRequests();
+
     // Initialise the update of interfaced interfaces
     initMatrixInterfaces
     (
@@ -274,7 +282,8 @@ void Foam::lduMatrix::residual
         interfaces,
         psi,
         rA,
-        cmpt
+        cmpt,
+        startRequest
     );
 }
 

src/OpenFOAM/matrices/lduMatrix/lduMatrix/lduMatrixUpdateMatrixInterfaces.C

@@ -106,7 +106,8 @@ void Foam::lduMatrix::updateMatrixInterfaces
     const lduInterfaceFieldPtrsList& interfaces,
     const solveScalarField& psiif,
     solveScalarField& result,
-    const direction cmpt
+    const direction cmpt,
+    const label startRequest
 ) const
 {
     if (Pstream::defaultCommsType == Pstream::commsTypes::blocking)
@@ -183,7 +184,7 @@ void Foam::lduMatrix::updateMatrixInterfaces
             else
             {
                 // Block for all requests and remove storage
-                UPstream::waitRequests();
+                UPstream::waitRequests(startRequest);
             }
         }
 

src/OpenFOAM/matrices/lduMatrix/smoothers/GaussSeidel/GaussSeidelSmoother.C

@@ -115,6 +115,8 @@ void Foam::GaussSeidelSmoother::smooth
     {
         bPrime = source;
 
+        const label startRequest = Pstream::nRequests();
+
         matrix_.initMatrixInterfaces
         (
             false,
@@ -132,7 +134,8 @@ void Foam::GaussSeidelSmoother::smooth
             interfaces_,
             psi,
             bPrime,
-            cmpt
+            cmpt,
+            startRequest
         );
 
         solveScalar psii;

src/OpenFOAM/matrices/lduMatrix/smoothers/nonBlockingGaussSeidel/nonBlockingGaussSeidelSmoother.C

@@ -142,6 +142,8 @@ void Foam::nonBlockingGaussSeidelSmoother::smooth
     {
         bPrime = source;
 
+        const label startRequest = Pstream::nRequests();
+
         matrix_.initMatrixInterfaces
         (
             false,
@@ -190,7 +192,8 @@ void Foam::nonBlockingGaussSeidelSmoother::smooth
             interfaces_,
             psi,
             bPrime,
-            cmpt
+            cmpt,
+            startRequest
         );
 
         // Update rest of the cells

src/OpenFOAM/matrices/lduMatrix/smoothers/symGaussSeidel/symGaussSeidelSmoother.C

@@ -115,6 +115,8 @@ void Foam::symGaussSeidelSmoother::smooth
     {
         bPrime = source;
 
+        const label startRequest = Pstream::nRequests();
+
         matrix_.initMatrixInterfaces
         (
             false,
@@ -132,7 +134,8 @@ void Foam::symGaussSeidelSmoother::smooth
             interfaces_,
             psi,
             bPrime,
-            cmpt
+            cmpt,
+            startRequest
         );
 
         solveScalar psii;

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverInterpolate.C

@@ -52,6 +52,8 @@ void Foam::GAMGSolver::interpolate
     Apsi = 0;
     solveScalar* __restrict__ ApsiPtr = Apsi.begin();
 
+    const label startRequest = Pstream::nRequests();
+
     m.initMatrixInterfaces
     (
         true,
@@ -76,7 +78,8 @@ void Foam::GAMGSolver::interpolate
         interfaces,
         psi,
         Apsi,
-        cmpt
+        cmpt,
+        startRequest
     );
 
     const label nCells = m.diag().size();

src/OpenFOAM/matrices/lduMatrix/solvers/PPCG/PPCG.C

@@ -0,0 +1,299 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | www.openfoam.com
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+    Copyright (C) 2019-2020 M. Janssens
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "PPCG.H"
+#include "PrecisionAdaptor.H"
+
+// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
+
+namespace Foam
+{
+    defineTypeNameAndDebug(PPCG, 0);
+
+    lduMatrix::solver::addsymMatrixConstructorToTable<PPCG>
+        addPPCGSymMatrixConstructorToTable_;
+}
+
+
+// * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
+
+void Foam::PPCG::gSumMagProd
+(
+    FixedList<solveScalar, 3>& globalSum,
+    const solveScalarField& a,
+    const solveScalarField& b,
+    const solveScalarField& c,
+    const solveScalarField& sumMag,
+    label& outstandingRequest,
+    const label comm
+) const
+{
+    const label nCells = a.size();
+
+    globalSum = 0.0;
+    for (label cell=0; cell<nCells; cell++)
+    {
+        globalSum[0] += a[cell]*b[cell];    // sumProd(a, b)
+        globalSum[1] += a[cell]*c[cell];    // sumProd(a, c)
+        globalSum[2] += mag(sumMag[cell]);
+    }
+
+    if (Pstream::parRun())
+    {
+        Foam::reduce
+        (
+            globalSum.begin(),
+            globalSum.size(),
+            sumOp<solveScalar>(),
+            Pstream::msgType(),
+            comm,
+            outstandingRequest
+        );
+    }
+}
+
+
+Foam::solverPerformance Foam::PPCG::scalarSolve
+(
+    solveScalarField& psi,
+    const solveScalarField& source,
+    const direction cmpt,
+    const bool cgMode
+) const
+{
+    // --- Setup class containing solver performance data
+    solverPerformance solverPerf
+    (
+        lduMatrix::preconditioner::getName(controlDict_) + type(),
+        fieldName_
+    );
+
+    const label comm = matrix().mesh().comm();
+    const label nCells = psi.size();
+    solveScalarField w(nCells);
+
+    // --- Calculate A.psi
+    matrix_.Amul(w, psi, interfaceBouCoeffs_, interfaces_, cmpt);
+
+    // --- Calculate initial residual field
+    solveScalarField r(source - w);
+
+    // --- Calculate normalisation factor
+    solveScalarField p(nCells);
+    const solveScalar normFactor = this->normFactor(psi, source, w, p);
+
+    if (lduMatrix::debug >= 2)
+    {
+        Info<< "   Normalisation factor = " << normFactor << endl;
+    }
+
+    // --- Select and construct the preconditioner
+    autoPtr<lduMatrix::preconditioner> preconPtr =
+    lduMatrix::preconditioner::New
+    (
+        *this,
+        controlDict_
+    );
+
+    // --- Precondition residual (= u0)
+    solveScalarField u(nCells);
+    preconPtr->precondition(u, r, cmpt);
+
+    // --- Calculate A*u - reuse w
+    matrix_.Amul(w, u, interfaceBouCoeffs_, interfaces_, cmpt);
+
+
+    // State
+    solveScalarField s(nCells);
+    solveScalarField q(nCells);
+    solveScalarField z(nCells);
+
+    solveScalarField m(nCells);
+
+    FixedList<solveScalar, 3> globalSum;
+    label outstandingRequest = -1;
+    if (cgMode)
+    {
+        // --- Start global reductions for inner products
+        gSumMagProd(globalSum, u, r, w, r, outstandingRequest, comm);
+
+        // --- Precondition residual
+        preconPtr->precondition(m, w, cmpt);
+    }
+    else
+    {
+        // --- Precondition residual
+        preconPtr->precondition(m, w, cmpt);
+
+        // --- Start global reductions for inner products
+        gSumMagProd(globalSum, w, u, m, r, outstandingRequest, comm);
+    }
+
+    // --- Calculate A*m
+    solveScalarField n(nCells);
+    matrix_.Amul(n, m, interfaceBouCoeffs_, interfaces_, cmpt);
+
+    solveScalar alpha = 0.0;
+    solveScalar gamma = 0.0;
+
+    // --- Solver iteration
+    for
+    (
+        solverPerf.nIterations() = 0;
+        solverPerf.nIterations() < maxIter_;
+        solverPerf.nIterations()++
+    )
+    {
+        // Make sure gamma,delta are available
+        if (Pstream::parRun())
+        {
+            Pstream::waitRequest(outstandingRequest);
+            outstandingRequest = -1;
+        }
+
+        const solveScalar gammaOld = gamma;
+        gamma = globalSum[0];
+        const solveScalar delta = globalSum[1];
+
+        solverPerf.finalResidual() = globalSum[2]/normFactor;
+        if (solverPerf.nIterations() == 0)
+        {
+            solverPerf.initialResidual() = solverPerf.finalResidual();
+        }
+
+        // Check convergence (bypass if not enough iterations yet)
+        if
+        (
+            (minIter_ <= 0 || solverPerf.nIterations() >= minIter_)
+         && solverPerf.checkConvergence(tolerance_, relTol_)
+        )
+        {
+            break;
+        }
+
+
+        if (solverPerf.nIterations() == 0)
+        {
+            alpha = gamma/delta;
+            z = n;
+            q = m;
+            s = w;
+            p = u;
+        }
+        else
+        {
+            const solveScalar beta = gamma/gammaOld;
+            alpha = gamma/(delta-beta*gamma/alpha);
+
+            for (label cell=0; cell<nCells; cell++)
+            {
+                z[cell] = n[cell] + beta*z[cell];
+                q[cell] = m[cell] + beta*q[cell];
+                s[cell] = w[cell] + beta*s[cell];
+                p[cell] = u[cell] + beta*p[cell];
+            }
+        }
+
+        for (label cell=0; cell<nCells; cell++)
+        {
+            psi[cell] += alpha*p[cell];
+            r[cell] -= alpha*s[cell];
+            u[cell] -= alpha*q[cell];
+            w[cell] -= alpha*z[cell];
+        }
+
+        if (cgMode)
+        {
+            // --- Start global reductions for inner products
+            gSumMagProd(globalSum, u, r, w, r, outstandingRequest, comm);
+
+            // --- Precondition residual
+            preconPtr->precondition(m, w, cmpt);
+        }
+        else
+        {
+            // --- Precondition residual
+            preconPtr->precondition(m, w, cmpt);
+
+            // --- Start global reductions for inner products
+            gSumMagProd(globalSum, w, u, m, r, outstandingRequest, comm);
+        }
+
+        // --- Calculate A*m
+        matrix_.Amul(n, m, interfaceBouCoeffs_, interfaces_, cmpt);
+    }
+
+    return solverPerf;
+}
+
+
+
+// * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
+
+Foam::PPCG::PPCG
+(
+    const word& fieldName,
+    const lduMatrix& matrix,
+    const FieldField<Field, scalar>& interfaceBouCoeffs,
+    const FieldField<Field, scalar>& interfaceIntCoeffs,
+    const lduInterfaceFieldPtrsList& interfaces,
+    const dictionary& solverControls
+)
+:
+    lduMatrix::solver
+    (
+        fieldName,
+        matrix,
+        interfaceBouCoeffs,
+        interfaceIntCoeffs,
+        interfaces,
+        solverControls
+    )
+{}
+
+
+// * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
+
+Foam::solverPerformance Foam::PPCG::solve
+(
+    scalarField& psi_s,
+    const scalarField& source,
+    const direction cmpt
+) const
+{
+    PrecisionAdaptor<solveScalar, scalar> tpsi(psi_s);
+    return scalarSolve
+    (
+        tpsi.ref(),
+        ConstPrecisionAdaptor<solveScalar, scalar>(source)(),
+        cmpt,
+        true   // operate in conjugate-gradient mode
+    );
+}
+
+
+// ************************************************************************* //

src/OpenFOAM/matrices/lduMatrix/solvers/PPCG/PPCG.H

@@ -0,0 +1,147 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | www.openfoam.com
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+    Copyright (C) 2019-2020 M. Janssens
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+Class
+    Foam::PPCG
+
+Description
+    Preconditioned pipelined conjugate gradient solver for symmetric
+    lduMatrices using a run-time selectable preconditioner.
+
+    Reference:
+    \verbatim
+        P. Ghysels, W. Vanroose.
+        "Hiding global synchronization latency in the
+         preconditioned Conjugate Gradient algorithm"
+    \endverbatim
+    and implementation details from
+    \verbatim
+        Paul Eller, William Gropp
+        "Scalable Non-blocking Preconditioned Conjugate Gradient Methods"
+    \endverbatim
+
+SourceFiles
+    PPCG.C
+
+\*---------------------------------------------------------------------------*/
+
+#ifndef PPCG_H
+#define PPCG_H
+
+#include "lduMatrix.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+/*---------------------------------------------------------------------------*\
+                           Class PPCG Declaration
+\*---------------------------------------------------------------------------*/
+
+class PPCG
+:
+    public lduMatrix::solver
+{
+    // Private Member Functions
+
+        //- Non-blocking version of sum(a*b), sum(a*c), sum(mag(sumMag))
+        void gSumMagProd
+        (
+            FixedList<solveScalar, 3>& globalSum,
+            const solveScalarField& a,
+            const solveScalarField& b,
+            const solveScalarField& c,
+            const solveScalarField& sumMag,
+            label& outstandingRequest,
+            const label comm
+        ) const;
+
+        //- Disallow default bitwise copy construct
+        PPCG(const PPCG&);
+
+        //- Disallow default bitwise assignment
+        void operator=(const PPCG&);
+
+
+protected:
+
+        //- CG solver. Operates either in conjugate-gradient mode or
+        //  conjugate residual
+        solverPerformance scalarSolve
+        (
+            solveScalarField& psi,
+            const solveScalarField& source,
+            const direction cmpt,
+            const bool cgMode
+        ) const;
+
+
+public:
+
+    //- Runtime type information
+    TypeName("PPCG");
+
+
+    // Constructors
+
+        //- Construct from matrix components and solver controls
+        PPCG
+        (
+            const word& fieldName,
+            const lduMatrix& matrix,
+            const FieldField<Field, scalar>& interfaceBouCoeffs,
+            const FieldField<Field, scalar>& interfaceIntCoeffs,
+            const lduInterfaceFieldPtrsList& interfaces,
+            const dictionary& solverControls
+        );
+
+
+    //- Destructor
+    virtual ~PPCG()
+    {}
+
+
+    // Member Functions
+
+        //- Solve the matrix with this solver
+        virtual solverPerformance solve
+        (
+            scalarField& psi,
+            const scalarField& source,
+            const direction cmpt=0
+        ) const;
+};
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace Foam
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+#endif
+
+// ************************************************************************* //

src/OpenFOAM/matrices/lduMatrix/solvers/PPCR/PPCR.C

@@ -0,0 +1,86 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | www.openfoam.com
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+    Copyright (C) 2019-2020 M. Janssens
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "PPCR.H"
+#include "PrecisionAdaptor.H"
+
+// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
+
+namespace Foam
+{
+    defineTypeNameAndDebug(PPCR, 0);
+
+    lduMatrix::solver::addsymMatrixConstructorToTable<PPCR>
+        addPPCRSymMatrixConstructorToTable_;
+}
+
+
+// * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
+
+Foam::PPCR::PPCR
+(
+    const word& fieldName,
+    const lduMatrix& matrix,
+    const FieldField<Field, scalar>& interfaceBouCoeffs,
+    const FieldField<Field, scalar>& interfaceIntCoeffs,
+    const lduInterfaceFieldPtrsList& interfaces,
+    const dictionary& solverControls
+)
+:
+    PPCG
+    (
+        fieldName,
+        matrix,
+        interfaceBouCoeffs,
+        interfaceIntCoeffs,
+        interfaces,
+        solverControls
+    )
+{}
+
+
+// * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
+
+Foam::solverPerformance Foam::PPCR::solve
+(
+    scalarField& psi_s,
+    const scalarField& source,
+    const direction cmpt
+) const
+{
+    PrecisionAdaptor<solveScalar, scalar> tpsi(psi_s);
+    return PPCG::scalarSolve
+    (
+        tpsi.ref(),
+        ConstPrecisionAdaptor<solveScalar, scalar>(source)(),
+        cmpt,
+        false   // operate in residual mode
+    );
+}
+
+
+// ************************************************************************* //

src/OpenFOAM/matrices/lduMatrix/solvers/PPCR/PPCR.H

@@ -0,0 +1,125 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | www.openfoam.com
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+    Copyright (C) 2019-2020 M. Janssens
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+Class
+    Foam::PPCR
+
+Description
+    Preconditioned pipelined conjugate residuals solver for symmetric
+    lduMatrices using a run-time selectable preconditioner.
+
+    Reference:
+    \verbatim
+        P. Ghysels, W. Vanroose.
+        "Hiding global synchronization latency in the
+         preconditioned Conjugate Gradient algorithm"
+    \endverbatim
+    and implementation details from
+    \verbatim
+        Paul Eller, William Gropp
+        "Scalable Non-blocking Preconditioned Conjugate Gradient Methods"
+    \endverbatim
+
+See also
+    PPCG
+
+SourceFiles
+    PPCR.C
+
+\*---------------------------------------------------------------------------*/
+
+#ifndef PPCR_H
+#define PPCR_H
+
+#include "PPCG.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+/*---------------------------------------------------------------------------*\
+                           Class PPCR Declaration
+\*---------------------------------------------------------------------------*/
+
+class PPCR
+:
+    public PPCG
+{
+    // Private Member Functions
+
+        //- Disallow default bitwise copy construct
+        PPCR(const PPCR&);
+
+        //- Disallow default bitwise assignment
+        void operator=(const PPCR&);
+
+
+public:
+
+    //- Runtime type information
+    TypeName("PPCR");
+
+
+    // Constructors
+
+        //- Construct from matrix components and solver controls
+        PPCR
+        (
+            const word& fieldName,
+            const lduMatrix& matrix,
+            const FieldField<Field, scalar>& interfaceBouCoeffs,
+            const FieldField<Field, scalar>& interfaceIntCoeffs,
+            const lduInterfaceFieldPtrsList& interfaces,
+            const dictionary& solverControls
+        );
+
+
+    //- Destructor
+    virtual ~PPCR()
+    {}
+
+
+    // Member Functions
+
+        //- Solve the matrix with this solver
+        virtual solverPerformance solve
+        (
+            scalarField& psi,
+            const scalarField& source,
+            const direction cmpt=0
+        ) const;
+};
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace Foam
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+#endif
+
+// ************************************************************************* //

src/Pstream/dummy/UPstream.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2018 OpenFOAM Foundation
-    Copyright (C) 2016-2019 OpenCFD Ltd.
+    Copyright (C) 2016-2020 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -96,6 +96,18 @@ void Foam::reduce(scalar&, const sumOp<scalar>&, const int, const label, label&)
 {}
 
 
+void Foam::reduce
+(
+    scalar[],
+    const int,
+    const sumOp<scalar>&,
+    const int,
+    const label,
+    label&
+)
+{}
+
+
 #if defined(WM_SPDP)
 void Foam::reduce
 (
@@ -138,9 +150,20 @@ void Foam::reduce
     label& request
 )
 {}
+void Foam::reduce
+(
+    solveScalar[],
+    const int,
+    const sumOp<solveScalar>&,
+    const int,
+    const label,
+    label&
+)
+{}
 #endif
 
 
+
 void Foam::UPstream::allToAll
 (
     const labelUList& sendData,

src/Pstream/mpi/PstreamGlobals.C

@@ -30,6 +30,7 @@ License
 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
 
 Foam::DynamicList<MPI_Request> Foam::PstreamGlobals::outstandingRequests_;
+Foam::DynamicList<Foam::label> Foam::PstreamGlobals::freedRequests_;
 
 int Foam::PstreamGlobals::nTags_ = 0;
 

src/Pstream/mpi/PstreamGlobals.H

@@ -50,6 +50,7 @@ namespace PstreamGlobals
 
 //- Outstanding non-blocking operations.
 extern DynamicList<MPI_Request> outstandingRequests_;
+extern DynamicList<label> freedRequests_;
 
 //- Max outstanding message tag operations.
 extern int nTags_;

src/Pstream/mpi/UPstream.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2017 OpenFOAM Foundation
-    Copyright (C) 2016-2019 OpenCFD Ltd.
+    Copyright (C) 2016-2020 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -307,18 +307,29 @@ void Foam::UPstream::exit(int errnum)
     }
 
 
-    const label nOutstanding = PstreamGlobals::outstandingRequests_.size();
-    if (nOutstanding)
+    if (PstreamGlobals::outstandingRequests_.size())
     {
+        label nOutstanding = 0;
+        forAll(PstreamGlobals::outstandingRequests_, i)
+        {
+            if (findIndex(PstreamGlobals::freedRequests_, i) == -1)
+            {
+                nOutstanding++;
+            }
+        }
+
         PstreamGlobals::outstandingRequests_.clear();
 
-        WarningInFunction
-            << "There were still " << nOutstanding
-            << " outstanding MPI_Requests." << nl
-            << "Which means your code exited before doing a "
-            << " UPstream::waitRequests()." << nl
-            << "This should not happen for a normal code exit."
-            << nl;
+        if (nOutstanding)
+        {
+            WarningInFunction
+                << "There were still " << nOutstanding
+                << " outstanding MPI_Requests." << nl
+                << "Which means your code exited before doing a "
+                << " UPstream::waitRequests()." << nl
+                << "This should not happen for a normal code exit."
+                << nl;
+        }
     }
 
     // Clean mpi communicators
@@ -449,37 +460,29 @@ void Foam::reduce
     label& requestID
 )
 {
-#ifdef MPIX_COMM_TYPE_SHARED
-    // Assume mpich2 with non-blocking collectives extensions. Once mpi3
-    // is available this will change.
-    MPI_Request request;
-    scalar v = Value;
-    MPIX_Ireduce
+    iallReduce<scalar>(&Value, 1, MPI_SCALAR, MPI_SUM, communicator, requestID);
+}
+
+
+void Foam::reduce
+(
+    scalar values[],
+    const int size,
+    const sumOp<scalar>& bop,
+    const int tag,
+    const label communicator,
+    label& requestID
+)
+{
+    iallReduce<scalar>
     (
-        &v,
-        &Value,
-        1,
+        values,
+        size,
         MPI_SCALAR,
         MPI_SUM,
-        0,              //root
-        PstreamGlobals::MPICommunicators_[communicator],
-        &request
+        communicator,
+        requestID
     );
-
-    requestID = PstreamGlobals::outstandingRequests_.size();
-    PstreamGlobals::outstandingRequests_.append(request);
-
-    if (UPstream::debug)
-    {
-        Pout<< "UPstream::allocateRequest for non-blocking reduce"
-            << " : request:" << requestID
-            << endl;
-    }
-#else
-    // Non-blocking not yet implemented in mpi
-    reduce(Value, bop, tag, communicator);
-    requestID = -1;
-#endif
 }
 
 
@@ -573,42 +576,41 @@ void Foam::reduce
     label& requestID
 )
 {
-#ifdef MPIX_COMM_TYPE_SHARED
-    // Assume mpich2 with non-blocking collectives extensions. Once mpi3
-    // is available this will change.
-    MPI_Request request;
-    solveScalar v = Value;
-    MPIX_Ireduce
+    iallReduce<solveScalar>
     (
-        &v,
         &Value,
         1,
         MPI_SOLVESCALAR,
         MPI_SUM,
-        0,              //root
-        PstreamGlobals::MPICommunicators_[communicator],
-        &request
+        communicator,
+        requestID
     );
+}
 
-    requestID = PstreamGlobals::outstandingRequests_.size();
-    PstreamGlobals::outstandingRequests_.append(request);
 
-    if (UPstream::debug)
-    {
-        Pout<< "UPstream::allocateRequest for non-blocking reduce"
-            << " : request:" << requestID
-            << endl;
-    }
-#else
-    // Non-blocking not yet implemented in mpi
-    reduce(Value, bop, tag, communicator);
-    requestID = -1;
-#endif
+void Foam::reduce
+(
+    solveScalar values[],
+    const int size,
+    const sumOp<solveScalar>& bop,
+    const int tag,
+    const label communicator,
+    label& requestID
+)
+{
+    iallReduce<solveScalar>
+    (
+        values,
+        size,
+        MPI_SOLVESCALAR,
+        MPI_SUM,
+        communicator,
+        requestID
+    );
 }
 #endif
 
 
-
 void Foam::UPstream::allToAll
 (
     const labelUList& sendData,
@@ -1054,7 +1056,7 @@ void Foam::UPstream::waitRequest(const label i)
             << endl;
     }
 
-    if (i >= PstreamGlobals::outstandingRequests_.size())
+    if (i < 0 || i >= PstreamGlobals::outstandingRequests_.size())
     {
         FatalErrorInFunction
             << "There are " << PstreamGlobals::outstandingRequests_.size()
@@ -1080,6 +1082,8 @@ void Foam::UPstream::waitRequest(const label i)
     }
 
     profilingPstream::addWaitTime();
+    // Push index onto free cache
+    PstreamGlobals::freedRequests_.append(i);
 
     if (debug)
     {

src/Pstream/mpi/allReduce.H

@@ -58,6 +58,17 @@ void allReduce
     const label communicator
 );
 
+template<class Type>
+void iallReduce
+(
+    void* Value,
+    int count,
+    MPI_Datatype MPIType,
+    MPI_Op op,
+    const label communicator,
+    label& requestID
+);
+
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
 } // End namespace Foam

src/Pstream/mpi/allReduceTemplates.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2012-2015 OpenFOAM Foundation
-    Copyright (C) 2019 OpenCFD Ltd.
+    Copyright (C) 2019-2020 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -176,4 +176,93 @@ void Foam::allReduce
 }
 
 
+template<class Type>
+void Foam::iallReduce
+(
+    void* recvBuf,
+    int MPICount,
+    MPI_Datatype MPIType,
+    MPI_Op MPIOp,
+    const label communicator,
+    label& requestID
+)
+{
+    if (!UPstream::parRun())
+    {
+        return;
+    }
+
+    if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
+    {
+        Pout<< "** non-blocking reducing:" << UList<Type>(recvBuf, MPICount)
+            << " with comm:" << communicator
+            << " warnComm:" << UPstream::warnComm << endl;
+        error::printStack(Pout);
+    }
+
+    profilingPstream::beginTiming();
+
+#if defined(MPI_VERSION) && (MPI_VERSION >= 3)
+    MPI_Request request;
+    if
+    (
+        MPI_Iallreduce
+        (
+            MPI_IN_PLACE,
+            recvBuf,
+            MPICount,
+            MPIType,
+            MPIOp,
+            PstreamGlobals::MPICommunicators_[communicator],
+            &request
+        )
+    )
+    {
+        FatalErrorInFunction
+            << "MPI_Iallreduce failed for "<< UList<Type>(recvBuf, MPICount)
+            << Foam::abort(FatalError);
+    }
+
+    if (PstreamGlobals::freedRequests_.size())
+    {
+        requestID = PstreamGlobals::freedRequests_.remove();
+        PstreamGlobals::outstandingRequests_[requestID] = request;
+    }
+    else
+    {
+        requestID = PstreamGlobals::outstandingRequests_.size();
+        PstreamGlobals::outstandingRequests_.append(request);
+    }
+
+    if (UPstream::debug)
+    {
+        Pout<< "UPstream::allocateRequest for non-blocking reduce"
+            << " : request:" << requestID << endl;
+    }
+#else
+    // Non-blocking not yet implemented in mpi
+    if
+    (
+        MPI_Allreduce
+        (
+            MPI_IN_PLACE,
+            recvBuf,
+            MPICount,
+            MPIType,
+            MPIOp,
+            PstreamGlobals::MPICommunicators_[communicator]
+        )
+    )
+    {
+        FatalErrorInFunction
+            << "MPI_Allreduce failed for " << UList<Type>(recvBuf, MPICount)
+            << Foam::abort(FatalError);
+    }
+    requestID = -1;
+#endif
+
+    profilingPstream::addReduceTime();
+}
+
+
 // ************************************************************************* //

src/finiteArea/faMatrices/faMatrix/faMatrixSolve.C

@@ -105,6 +105,8 @@ Foam::SolverPerformance<Type> Foam::faMatrix<Type>::solve
             PrecisionAdaptor<solveScalar, scalar> sourceCmpt_ss(sourceCmpt);
             ConstPrecisionAdaptor<solveScalar, scalar> psiCmpt_ss(psiCmpt);
 
+            const label startRequest = Pstream::nRequests();
+
             initMatrixInterfaces
             (
                 true,
@@ -122,7 +124,8 @@ Foam::SolverPerformance<Type> Foam::faMatrix<Type>::solve
                 interfaces,
                 psiCmpt_ss(),
                 sourceCmpt_ss.ref(),
-                cmpt
+                cmpt,
+                startRequest
             );
         }
 

src/finiteVolume/fvMatrices/fvMatrix/fvMatrixSolve.C

@@ -179,6 +179,8 @@ Foam::SolverPerformance<Type> Foam::fvMatrix<Type>::solveSegregated
             PrecisionAdaptor<solveScalar, scalar> sourceCmpt_ss(sourceCmpt);
             ConstPrecisionAdaptor<solveScalar, scalar> psiCmpt_ss(psiCmpt);
 
+            const label startRequest = Pstream::nRequests();
+
             initMatrixInterfaces
             (
                 true,
@@ -196,7 +198,8 @@ Foam::SolverPerformance<Type> Foam::fvMatrix<Type>::solveSegregated
                 interfaces,
                 psiCmpt_ss(),
                 sourceCmpt_ss.ref(),
-                cmpt
+                cmpt,
+                startRequest
             );
         }