ENH: comunicators: have solver with user-defined communicator

(originating from polyMesh::comm() and processorPolyPatch::comm())

src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2012 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -48,6 +48,12 @@ void reduce
     const label comm
 )
 {
+    if (UPstream::warnComm != -1 && comm != UPstream::warnComm)
+    {
+        Pout<< "** reducing:" << Value << " with comm:" << comm
+            << endl;
+        error::printStack(Pout);
+    }
     Pstream::gather(comms, Value, bop, tag, comm);
     Pstream::scatter(comms, Value, tag, comm);
 }

src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2012 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2012-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -270,6 +270,19 @@ Foam::label Foam::UPstream::allocateCommunicator
     forAll(procIDs_[index], i)
     {
         procIDs_[index][i] = subRanks[i];
+
+        // Enforce incremental order (so index is rank in next communicator)
+        if (i >= 1 && subRanks[i] <= subRanks[i-1])
+        {
+            FatalErrorIn
+            (
+                "UPstream::allocateCommunicator"
+                "(const label, const labelList&, const bool)"
+            )   << "subranks not sorted : " << subRanks
+                << " when allocating subcommunicator from parent "
+                << parentIndex
+                << Foam::abort(FatalError);
+        }
     }
     parentCommunicator_[index] = parentIndex;
 
@@ -277,7 +290,7 @@ Foam::label Foam::UPstream::allocateCommunicator
     treeCommunication_[index] = calcTreeComm(procIDs_[index].size());
 
 
-    if (doPstream)
+    if (doPstream && parRun())
     {
         allocatePstreamCommunicator(parentIndex, index);
     }
@@ -297,7 +310,7 @@ void Foam::UPstream::freeCommunicator
         << "    myProcNo : " << myProcNo_[communicator] << endl
         << endl;
 
-    if (doPstream)
+    if (doPstream && parRun())
     {
         freePstreamCommunicator(communicator);
     }
@@ -437,6 +450,10 @@ addcommsTypeToOpt addcommsTypeToOpt_("commsType");
 Foam::label Foam::UPstream::worldComm(0);
 
 
+// Warn for use of any communicator
+Foam::label Foam::UPstream::warnComm(-1);
+
+
 // Number of polling cycles in processor updates
 int Foam::UPstream::nPollProcInterfaces
 (

src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2012 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2012-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -269,6 +269,10 @@ public:
         //- Default communicator (all processors)
         static label worldComm;
 
+        //- Debugging: warn for use of any communicator differing from warnComm
+        static label warnComm;
+
+
     // Constructors
 
         //- Construct given optional buffer size
@@ -319,17 +323,17 @@ public:
                 comm_(allocateCommunicator(parent, subRanks, doPstream))
             {}
 
-            communicator(const label parent)
+//            communicator(const label parent)
-            :
+//            :
-                comm_
+//                comm_
-                (
+//                (
-                    allocateCommunicator
+//                    allocateCommunicator
-                    (
+//                    (
-                        parent,
+//                        parent,
-                        identity(UPstream::nProcs(parent))
+//                        identity(UPstream::nProcs(parent))
-                    )
+//                    )
-                )
+//                )
-            {}
+//            {}
 
             ~communicator()
             {
@@ -407,17 +411,24 @@ public:
             return myProcNo_[communicator];
         }
 
+        static label parent(const label communicator)
+        {
+            return parentCommunicator_(communicator);
+        }
+
+
 //        //- Process IDs
 //        static const List<int>& procIDs()
 //        {
 //            return procIDs_;
 //        }
-//
+
-//        //- Process ID of given process index
+
-//        static int procID(int procNo)
+        //- Process ID of given process index
-//        {
+        static List<int>& procID(int communicator)
-//            return procIDs_[procNo];
+        {
-//        }
+            return procIDs_[communicator];
+        }
 
         //- Process index of first slave
         static int firstSlave()

src/OpenFOAM/db/error/messageStream.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -166,6 +166,74 @@ Foam::OSstream& Foam::messageStream::operator()
 }
 
 
+Foam::OSstream& Foam::messageStream::operator()(const label communicator)
+{
+    if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
+    {
+        Pout<< "** messageStream with comm:" << communicator
+            << endl;
+        error::printStack(Pout);
+    }
+
+    if (communicator == UPstream::worldComm)
+    {
+        return operator()();
+    }
+    else
+    {
+        bool master = UPstream::master(communicator);
+
+        if (level)
+        {
+            bool collect = (severity_ == INFO || severity_ == WARNING);
+
+            // Report the error
+            if (!master && collect)
+            {
+                return Snull;
+            }
+            else
+            {
+                if (title().size())
+                {
+                    if (Pstream::parRun() && !collect)
+                    {
+                        Pout<< title().c_str();
+                    }
+                    else
+                    {
+                        Sout<< title().c_str();
+                    }
+                }
+
+                if (maxErrors_)
+                {
+                    errorCount_++;
+
+                    if (errorCount_ >= maxErrors_)
+                    {
+                        FatalErrorIn("messageStream::operator OSstream&()")
+                            << "Too many errors"
+                            << abort(FatalError);
+                    }
+                }
+
+                if (Pstream::parRun() && !collect)
+                {
+                    return Pout;
+                }
+                else
+                {
+                    return Sout;
+                }
+            }
+        }
+    }
+
+    return Snull;
+}
+
+
 Foam::messageStream::operator Foam::OSstream&()
 {
     if (level)

src/OpenFOAM/db/error/messageStream.H

@@ -184,6 +184,11 @@ public:
             const dictionary&
         );
 
+        //- Convert to OSstream
+        //  Use Info for default communicator, use Pout
+        //  on master for non-default one.
+        OSstream& operator()(const label communicator);
+
         //- Convert to OSstream for << operations
         operator OSstream&();
 

src/OpenFOAM/fields/Fields/Field/FieldFunctions.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2012 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -477,10 +477,10 @@ TMP_UNARY_FUNCTION(Type, average)
 #define G_UNARY_FUNCTION(ReturnType, gFunc, Func, rFunc)                      \
                                                                               \
 template<class Type>                                                          \
-ReturnType gFunc(const UList<Type>& f)                                        \
+ReturnType gFunc(const UList<Type>& f, const int comm)                        \
 {                                                                             \
     ReturnType res = Func(f);                                                 \
-    reduce(res, rFunc##Op<Type>());                                           \
+    reduce(res, rFunc##Op<Type>(), Pstream::msgType(), comm);                 \
     return res;                                                               \
 }                                                                             \
 TMP_UNARY_FUNCTION(ReturnType, gFunc)
@@ -495,27 +495,41 @@ G_UNARY_FUNCTION(Type, gSumCmptMag, sumCmptMag, sum)
 #undef G_UNARY_FUNCTION
 
 template<class Type>
-scalar gSumProd(const UList<Type>& f1, const UList<Type>& f2)
+scalar gSumProd
+(
+    const UList<Type>& f1,
+    const UList<Type>& f2,
+    const int comm
+)
 {
     scalar SumProd = sumProd(f1, f2);
-    reduce(SumProd, sumOp<scalar>());
+    reduce(SumProd, sumOp<scalar>(), Pstream::msgType(), comm);
     return SumProd;
 }
 
 template<class Type>
-Type gSumCmptProd(const UList<Type>& f1, const UList<Type>& f2)
+Type gSumCmptProd
+(
+    const UList<Type>& f1,
+    const UList<Type>& f2,
+    const int comm
+)
 {
     Type SumProd = sumCmptProd(f1, f2);
-    reduce(SumProd, sumOp<Type>());
+    reduce(SumProd, sumOp<Type>(), Pstream::msgType(), comm);
     return SumProd;
 }
 
 template<class Type>
-Type gAverage(const UList<Type>& f)
+Type gAverage
+(
+    const UList<Type>& f,
+    const int comm
+)
 {
     label n = f.size();
     Type s = sum(f);
-    sumReduce(s, n);
+    sumReduce(s, n, Pstream::msgType(), comm);
 
     if (n > 0)
     {

src/OpenFOAM/fields/Fields/Field/FieldFunctions.H

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2012 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -25,6 +25,7 @@ License
 
 #define TEMPLATE template<class Type>
 #include "FieldFunctionsM.H"
+#include "UPstream.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
@@ -201,7 +202,7 @@ TMP_UNARY_FUNCTION(Type, average)
 #define G_UNARY_FUNCTION(ReturnType, gFunc, Func, rFunc)                      \
                                                                               \
 template<class Type>                                                          \
-ReturnType gFunc(const UList<Type>& f);                                       \
+ReturnType gFunc(const UList<Type>& f, const int comm = UPstream::worldComm); \
 TMP_UNARY_FUNCTION(ReturnType, gFunc)
 
 G_UNARY_FUNCTION(Type, gMax, max, max)
@@ -214,13 +215,27 @@ G_UNARY_FUNCTION(Type, gSumCmptMag, sumCmptMag, sum)
 #undef G_UNARY_FUNCTION
 
 template<class Type>
-scalar gSumProd(const UList<Type>& f1, const UList<Type>& f2);
+scalar gSumProd
+(
+    const UList<Type>& f1,
+    const UList<Type>& f2,
+    const int comm = UPstream::worldComm
+);
 
 template<class Type>
-Type gSumCmptProd(const UList<Type>& f1, const UList<Type>& f2);
+Type gSumCmptProd
+(
+    const UList<Type>& f1,
+    const UList<Type>& f2,
+    const int comm = UPstream::worldComm
+);
 
 template<class Type>
-Type gAverage(const UList<Type>& f);
+Type gAverage
+(
+    const UList<Type>& f,
+    const int comm = UPstream::worldComm
+);
 
 TMP_UNARY_FUNCTION(Type, gAverage)
 

src/OpenFOAM/matrices/LduMatrix/LduMatrix/SolverPerformance.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2012 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -24,6 +24,7 @@ License
 \*---------------------------------------------------------------------------*/
 
 #include "SolverPerformance.H"
+#include "IOstreams.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
@@ -64,7 +65,8 @@ bool Foam::SolverPerformance<Type>::checkConvergence
 {
     if (debug >= 2)
     {
-        Info<< solverName_
+        //Info<< solverName_
+        Pout<< solverName_
             << ":  Iteration " << noIterations_
             << " residual = " << finalResidual_
             << endl;

src/OpenFOAM/matrices/lduMatrix/lduAddressing/lduInterface/processorLduInterface.H

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -84,7 +84,7 @@ public:
 
         // Access
 
-            //- Return communicator used for sending
+            //- Return communicator used for parallel communication
             virtual int comm() const = 0;
 
             //- Return processor number (rank in communicator)

src/OpenFOAM/matrices/lduMatrix/lduMatrix/lduMatrixSolver.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2012 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -186,10 +186,15 @@ Foam::scalar Foam::lduMatrix::solver::normFactor
 {
     // --- Calculate A dot reference value of psi
     matrix_.sumA(tmpField, interfaceBouCoeffs_, interfaces_);
-    tmpField *= gAverage(psi);
+
+    tmpField *= gAverage(psi, matrix_.lduMesh_.comm());
 
     return
-        gSum(mag(Apsi - tmpField) + mag(source - tmpField))
+        gSum
+        (
+            (mag(Apsi - tmpField) + mag(source - tmpField))(),
+            matrix_.lduMesh_.comm()
+        )
       + solverPerformance::small_;
 
     // At convergence this simpler method is equivalent to the above

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -280,6 +280,7 @@ void Foam::GAMGAgglomeration::agglomerateLduAddressing
             coarseInterfaceAddr,
             coarseInterfaces,
             fineMeshAddr.patchSchedule(),
+            fineMesh.comm(),
             true
         )
     );

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomeration.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -57,7 +57,7 @@ bool Foam::GAMGAgglomeration::continueAgglomerating
 {
     // Check the need for further agglomeration on all processors
     bool contAgg = nCoarseCells >= nCellsInCoarsestLevel_;
-    reduce(contAgg, andOp<bool>());
+    mesh().reduce(contAgg, andOp<bool>());
     return contAgg;
 }
 

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomeration.H

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -51,6 +51,7 @@ namespace Foam
 
 class lduMesh;
 class lduMatrix;
+class mapDistribute;
 
 /*---------------------------------------------------------------------------*\
                     Class GAMGAgglomeration Declaration
@@ -77,6 +78,9 @@ protected:
         //  Maps from the finer to the coarser level.
         PtrList<labelField> restrictAddressing_;
 
+        //- For cross-processor operation
+        autoPtr<mapDistribute> restrictDistributeMapPtr_;
+
         //- Face restriction addressing array.
         //  Maps from the finer to the coarser level.
         //  Positive indices map the finer faces which form part of the boundary
@@ -85,6 +89,9 @@ protected:
         //  coarser cells to minus the corresponding coarser cell index minus 1.
         PtrList<labelList> faceRestrictAddressing_;
 
+        //- For cross-processor operation
+        autoPtr<mapDistribute> faceRestrictDistributeMapPtr_;
+
         //- Hierarchy of mesh addressing
         PtrList<lduPrimitiveMesh> meshLevels_;
 

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomerationTemplates.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -24,6 +24,7 @@ License
 \*---------------------------------------------------------------------------*/
 
 #include "GAMGAgglomeration.H"
+#include "mapDistribute.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
@@ -37,6 +38,38 @@ void Foam::GAMGAgglomeration::restrictField
 {
     const labelList& fineToCoarse = restrictAddressing_[fineLevelIndex];
 
+    if (restrictDistributeMapPtr_.valid())
+    {
+        const mapDistribute& map = restrictDistributeMapPtr_();
+        Field<Type> combinedFf(map.constructSize());
+        forAll(ff, i)
+        {
+            combinedFf[i] = ff[i];
+        }
+        map.distribute(combinedFf);
+
+        if (combinedFf.size() != fineToCoarse.size())
+        {
+            FatalErrorIn
+            (
+                "void GAMGAgglomeration::restrictField"
+                "(Field<Type>& cf, const Field<Type>& ff, "
+                "const label fineLevelIndex) const"
+            )   << "field does not correspond to level " << fineLevelIndex
+                << " sizes: field = " << combinedFf.size()
+                << " level = " << fineToCoarse.size()
+                << abort(FatalError);
+        }
+
+        cf = pTraits<Type>::zero;
+
+        forAll(combinedFf, i)
+        {
+            cf[fineToCoarse[i]] += combinedFf[i];
+        }
+    }
+    else
+    {
         if (ff.size() != fineToCoarse.size())
         {
             FatalErrorIn
@@ -57,6 +90,7 @@ void Foam::GAMGAgglomeration::restrictField
             cf[fineToCoarse[i]] += ff[i];
         }
     }
+}
 
 
 template<class Type>
@@ -69,6 +103,30 @@ void Foam::GAMGAgglomeration::restrictFaceField
 {
     const labelList& fineToCoarse = faceRestrictAddressing_[fineLevelIndex];
 
+    if (faceRestrictDistributeMapPtr_.valid())
+    {
+        const mapDistribute& map = faceRestrictDistributeMapPtr_();
+        Field<Type> combinedFf(map.constructSize());
+        forAll(ff, i)
+        {
+            combinedFf[i] = ff[i];
+        }
+        map.distribute(combinedFf);
+
+        cf = pTraits<Type>::zero;
+
+        forAll(fineToCoarse, ffacei)
+        {
+            label cFace = fineToCoarse[ffacei];
+
+            if (cFace >= 0)
+            {
+                cf[cFace] += combinedFf[ffacei];
+            }
+        }
+    }
+    else
+    {
         cf = pTraits<Type>::zero;
 
         forAll(fineToCoarse, ffacei)
@@ -81,6 +139,7 @@ void Foam::GAMGAgglomeration::restrictFaceField
             }
         }
     }
+}
 
 
 template<class Type>
@@ -93,11 +152,25 @@ void Foam::GAMGAgglomeration::prolongField
 {
     const labelList& fineToCoarse = restrictAddressing_[coarseLevelIndex];
 
+    if (restrictDistributeMapPtr_.valid())
+    {
+        const mapDistribute& map = restrictDistributeMapPtr_();
+        Field<Type> combinedCf(cf);
+        map.reverseDistribute(fineToCoarse.size(), combinedCf);
+
+        forAll(fineToCoarse, i)
+        {
+            ff[i] = combinedCf[fineToCoarse[i]];
+        }
+    }
+    else
+    {
         forAll(fineToCoarse, i)
         {
             ff[i] = cf[fineToCoarse[i]];
         }
     }
+}
 
 
 // ************************************************************************* //

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolver.H

@@ -43,9 +43,7 @@ Description
 
 SourceFiles
     GAMGSolver.C
-    GAMGSolverCalcAgglomeration.C
+    GAMGSolverAgglomerateMatrix.C
-    GAMGSolverMakeCoarseMatrix.C
-    GAMGSolverOperations.C
     GAMGSolverInterpolate.C
     GAMGSolverScale.C
     GAMGSolverSolve.C

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverAgglomerateMatrix.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -126,7 +126,7 @@ void Foam::GAMGSolver::agglomerateMatrix(const label fineLevelIndex)
     }
 
 
-    // Check if matrix is assymetric and if so agglomerate both upper and lower
+    // Check if matrix is asymetric and if so agglomerate both upper and lower
     // coefficients ...
     if (fineMatrix.hasLower())
     {

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverScale.C

@@ -58,7 +58,11 @@ void Foam::GAMGSolver::scale
     }
 
     vector2D scalingVector(scalingFactorNum, scalingFactorDenom);
-    reduce(scalingVector, sumOp<vector2D>());
+    matrix().mesh().reduce
+    (
+        scalingVector,
+        sumOp<vector2D>()
+    );
     scalar sf = scalingVector.x()/stabilise(scalingVector.y(), VSMALL);
 
     if (debug >= 2)

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverSolve.C

@@ -61,7 +61,11 @@ Foam::solverPerformance Foam::GAMGSolver::solve
     scalarField finestResidual(source - Apsi);
 
     // Calculate normalised residual for convergence test
-    solverPerf.initialResidual() = gSumMag(finestResidual)/normFactor;
+    solverPerf.initialResidual() = gSumMag
+    (
+        finestResidual,
+        matrix().mesh().comm()
+    )/normFactor;
     solverPerf.finalResidual() = solverPerf.initialResidual();
 
 
@@ -100,11 +104,15 @@ Foam::solverPerformance Foam::GAMGSolver::solve
             finestResidual = source;
             finestResidual -= Apsi;
 
-            solverPerf.finalResidual() = gSumMag(finestResidual)/normFactor;
+            solverPerf.finalResidual() = gSumMag
+            (
+                finestResidual,
+                matrix().mesh().comm()
+            )/normFactor;
 
             if (debug >= 2)
             {
-                solverPerf.print(Info);
+                solverPerf.print(Info(matrix().mesh().comm()));
             }
         } while
         (
@@ -481,7 +489,7 @@ void Foam::GAMGSolver::solveCoarsestLevel
 
         if (debug >= 2)
         {
-            coarseSolverPerf.print(Info);
+            coarseSolverPerf.print(Info(matrix().mesh().comm()));
         }
     }
 }

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaceFields/processorGAMGInterfaceField/processorGAMGInterfaceField.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2012 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2012-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -92,7 +92,8 @@ void Foam::processorGAMGInterfaceField::initInterfaceMatrixUpdate
             procInterface_.neighbProcNo(),
             reinterpret_cast<char*>(scalarReceiveBuf_.begin()),
             scalarReceiveBuf_.byteSize(),
-            procInterface_.tag()
+            procInterface_.tag(),
+            procInterface_.comm()
         );
 
         outstandingSendRequest_ = UPstream::nRequests();
@@ -102,7 +103,8 @@ void Foam::processorGAMGInterfaceField::initInterfaceMatrixUpdate
             procInterface_.neighbProcNo(),
             reinterpret_cast<const char*>(scalarSendBuf_.begin()),
             scalarSendBuf_.byteSize(),
-            procInterface_.tag()
+            procInterface_.tag(),
+            procInterface_.comm()
         );
     }
     else

src/OpenFOAM/matrices/lduMatrix/solvers/PBiCG/PBiCG.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2012 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -114,7 +114,9 @@ Foam::solverPerformance Foam::PBiCG::solve
     }
 
     // --- Calculate normalised residual norm
-    solverPerf.initialResidual() = gSumMag(rA)/normFactor;
+    solverPerf.initialResidual() =
+        gSumMag(rA, matrix().mesh().comm())
+       /normFactor;
     solverPerf.finalResidual() = solverPerf.initialResidual();
 
     // --- Check convergence, solve if not converged
@@ -139,7 +141,7 @@ Foam::solverPerformance Foam::PBiCG::solve
             preconPtr->preconditionT(wT, rT, cmpt);
 
             // --- Update search directions:
-            wArT = gSumProd(wA, rT);
+            wArT = gSumProd(wA, rT, matrix().mesh().comm());
 
             if (solverPerf.nIterations() == 0)
             {
@@ -165,7 +167,7 @@ Foam::solverPerformance Foam::PBiCG::solve
             matrix_.Amul(wA, pA, interfaceBouCoeffs_, interfaces_, cmpt);
             matrix_.Tmul(wT, pT, interfaceIntCoeffs_, interfaces_, cmpt);
 
-            scalar wApT = gSumProd(wA, pT);
+            scalar wApT = gSumProd(wA, pT, matrix().mesh().comm());
 
             // --- Test for singularity
             if (solverPerf.checkSingularity(mag(wApT)/normFactor))
@@ -185,8 +187,9 @@ Foam::solverPerformance Foam::PBiCG::solve
                 rTPtr[cell] -= alpha*wTPtr[cell];
             }
 
-            solverPerf.finalResidual() = gSumMag(rA)/normFactor;
+            solverPerf.finalResidual() =
-
+                gSumMag(rA, matrix().mesh().comm())
+               /normFactor;
         } while
         (
             solverPerf.nIterations()++ < maxIter_

src/OpenFOAM/matrices/lduMatrix/solvers/PCG/PCG.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2012 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -105,7 +105,9 @@ Foam::solverPerformance Foam::PCG::solve
     }
 
     // --- Calculate normalised residual norm
-    solverPerf.initialResidual() = gSumMag(rA)/normFactor;
+    solverPerf.initialResidual() =
+        gSumMag(rA, matrix().mesh().comm())
+       /normFactor;
     solverPerf.finalResidual() = solverPerf.initialResidual();
 
     // --- Check convergence, solve if not converged
@@ -129,7 +131,7 @@ Foam::solverPerformance Foam::PCG::solve
             preconPtr->precondition(wA, rA, cmpt);
 
             // --- Update search directions:
-            wArA = gSumProd(wA, rA);
+            wArA = gSumProd(wA, rA, matrix().mesh().comm());
 
             if (solverPerf.nIterations() == 0)
             {
@@ -152,7 +154,7 @@ Foam::solverPerformance Foam::PCG::solve
             // --- Update preconditioned residual
             matrix_.Amul(wA, pA, interfaceBouCoeffs_, interfaces_, cmpt);
 
-            scalar wApA = gSumProd(wA, pA);
+            scalar wApA = gSumProd(wA, pA, matrix().mesh().comm());
 
 
             // --- Test for singularity
@@ -169,7 +171,9 @@ Foam::solverPerformance Foam::PCG::solve
                 rAPtr[cell] -= alpha*wAPtr[cell];
             }
 
-            solverPerf.finalResidual() = gSumMag(rA)/normFactor;
+            solverPerf.finalResidual() =
+                gSumMag(rA, matrix().mesh().comm())
+               /normFactor;
 
         } while
         (

src/OpenFOAM/matrices/lduMatrix/solvers/smoothSolver/smoothSolver.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2012 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -122,13 +122,18 @@ Foam::solverPerformance Foam::smoothSolver::solve
             normFactor = this->normFactor(psi, source, Apsi, temp);
 
             // Calculate residual magnitude
-            solverPerf.initialResidual() = gSumMag(source - Apsi)/normFactor;
+            solverPerf.initialResidual() = gSumMag
+            (
+                (source - Apsi)(),
+                matrix().mesh().comm()
+            )/normFactor;
             solverPerf.finalResidual() = solverPerf.initialResidual();
         }
 
         if (lduMatrix::debug >= 2)
         {
-            Info<< "   Normalisation factor = " << normFactor << endl;
+            Info(matrix().mesh().comm())
+                << "   Normalisation factor = " << normFactor << endl;
         }
 
 
@@ -166,7 +171,8 @@ Foam::solverPerformance Foam::smoothSolver::solve
                         interfaceBouCoeffs_,
                         interfaces_,
                         cmpt
-                    )
+                    )(),
+                    matrix().mesh().comm()
                 )/normFactor;
             } while
             (

src/OpenFOAM/matrices/lduMatrix/solvers/smoothSolver/smoothSolver.H

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2012 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -25,7 +25,7 @@ Class
     Foam::smoothSolver
 
 Description
-    Iterative solver for symmetric and assymetric matrices which uses a
+    Iterative solver for symmetric and asymetric matrices which uses a
     run-time selected smoother e.g. GaussSeidel to converge the solution to
     the required tolerance.
 

src/OpenFOAM/meshes/lduMesh/lduMesh.H

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -77,6 +77,17 @@ public:
             //- Return a list of pointers for each patch
             //  with only those pointing to interfaces being set
             virtual lduInterfacePtrsList interfaces() const = 0;
+
+            //- Return communicator used for parallel communication
+            virtual int comm() const = 0;
+
+            //- Helper: reduce with current communicator
+            template<class T, class BinaryOp>
+            void reduce
+            (
+                T& Value,
+                const BinaryOp& bop
+            ) const;
 };
 
 
@@ -86,6 +97,12 @@ public:
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
+#ifdef NoRepository
+#   include "lduMeshTemplates.C"
+#endif
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
 #endif
 
 // ************************************************************************* //

src/OpenFOAM/meshes/lduMesh/lduMeshTemplates.C

@@ -0,0 +1,41 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | Copyright (C) 2013 OpenFOAM Foundation
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "lduMesh.H"
+
+// * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
+
+template<class T, class BinaryOp>
+void Foam::lduMesh::reduce
+(
+    T& Value,
+    const BinaryOp& bop
+) const
+{
+    Foam::reduce(Value, bop, Pstream::msgType(), comm());
+}
+
+
+// ************************************************************************* //

src/OpenFOAM/meshes/lduMesh/lduPrimitiveMesh.H

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -68,6 +68,8 @@ class lduPrimitiveMesh
         //  Note this does not need to be held as a copy because it is invariant
         const lduSchedule& patchSchedule_;
 
+        //- Communicator to use for any parallel communication
+        const label comm_;
 
     // Private Member Functions
 
@@ -90,7 +92,8 @@ public:
             const labelUList& u,
             const labelListList& pa,
             lduInterfacePtrsList interfaces,
-            const lduSchedule& ps
+            const lduSchedule& ps,
+            const label comm
         )
         :
             lduAddressing(nCells),
@@ -98,8 +101,18 @@ public:
             upperAddr_(u),
             patchAddr_(pa),
             interfaces_(interfaces),
-            patchSchedule_(ps)
+            patchSchedule_(ps),
-        {}
+            comm_(comm)
+        {
+            Pout<< "lduPrimitiveMesh :"
+                << " nCells:" << nCells
+                << " l:" << lowerAddr_.size()
+                << " u::" << upperAddr_.size()
+                << " pa:" << patchAddr_.size()
+                << " interfaces:" << interfaces_.size()
+                << " comm:" << comm_
+                << endl;
+        }
 
 
         //- Construct from components and  re-use storage as specified.
@@ -111,6 +124,7 @@ public:
             labelListList& pa,
             lduInterfacePtrsList interfaces,
             const lduSchedule& ps,
+            const label comm,
             bool reUse
         )
         :
@@ -119,8 +133,18 @@ public:
             upperAddr_(u, reUse),
             patchAddr_(pa, reUse),
             interfaces_(interfaces, reUse),
-            patchSchedule_(ps)
+            patchSchedule_(ps),
-        {}
+            comm_(comm)
+        {
+            Pout<< "lduPrimitiveMesh :"
+                << " nCells:" << nCells
+                << " l:" << lowerAddr_.size()
+                << " u::" << upperAddr_.size()
+                << " pa:" << patchAddr_.size()
+                << " interfaces:" << interfaces_.size()
+                << " comm:" << comm_
+                << endl;
+        }
 
 
     //- Destructor
@@ -145,6 +169,12 @@ public:
                 return interfaces_;
             }
 
+            //- Return communicator used for parallel communication
+            virtual label comm() const
+            {
+                return comm_;
+            }
+
             //- Return Lower addressing
             virtual const labelUList& lowerAddr() const
             {

src/OpenFOAM/meshes/polyMesh/polyMesh.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2012 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -210,6 +210,7 @@ Foam::polyMesh::polyMesh(const IOobject& io)
         *this
     ),
     bounds_(points_),
+    comm_(UPstream::worldComm),
     geometricD_(Vector<label>::zero),
     solutionD_(Vector<label>::zero),
     tetBasePtIsPtr_(NULL),
@@ -403,6 +404,7 @@ Foam::polyMesh::polyMesh
         polyPatchList()
     ),
     bounds_(points_, syncPar),
+    comm_(UPstream::worldComm),
     geometricD_(Vector<label>::zero),
     solutionD_(Vector<label>::zero),
     tetBasePtIsPtr_(NULL),
@@ -561,6 +563,7 @@ Foam::polyMesh::polyMesh
         0
     ),
     bounds_(points_, syncPar),
+    comm_(UPstream::worldComm),
     geometricD_(Vector<label>::zero),
     solutionD_(Vector<label>::zero),
     tetBasePtIsPtr_(NULL),
@@ -1212,6 +1215,18 @@ const Foam::globalMeshData& Foam::polyMesh::globalData() const
 }
 
 
+Foam::label Foam::polyMesh::comm() const
+{
+    return comm_;
+}
+
+
+Foam::label& Foam::polyMesh::comm()
+{
+    return comm_;
+}
+
+
 // Remove all files and some subdirs (eg, sets)
 void Foam::polyMesh::removeFiles(const fileName& instanceDir) const
 {

src/OpenFOAM/meshes/polyMesh/polyMesh.H

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2012 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -133,6 +133,9 @@ private:
             //  Created from points on construction, updated when the mesh moves
             boundBox bounds_;
 
+            //- Communicator used for parallel communication
+            label comm_;
+
             //- vector of non-constrained directions in mesh
             //  defined according to the presence of empty and wedge patches
             mutable Vector<label> geometricD_;
@@ -442,6 +445,12 @@ public:
             //- Return parallel info
             const globalMeshData& globalData() const;
 
+            //- Return communicator used for parallel communication
+            label comm() const;
+
+            //- Return communicator used for parallel communication
+            label& comm();
+
             //- Return the object registry
             const objectRegistry& thisDb() const
             {

src/OpenFOAM/meshes/polyMesh/polyMeshFromShapeMesh.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2012 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -504,6 +504,7 @@ Foam::polyMesh::polyMesh
         boundaryFaces.size() + 1    // add room for a default patch
     ),
     bounds_(points_, syncPar),
+    comm_(UPstream::worldComm),
     geometricD_(Vector<label>::zero),
     solutionD_(Vector<label>::zero),
     tetBasePtIsPtr_(NULL),
@@ -787,6 +788,7 @@ Foam::polyMesh::polyMesh
         boundaryFaces.size() + 1    // add room for a default patch
     ),
     bounds_(points_, syncPar),
+    comm_(UPstream::worldComm),
     geometricD_(Vector<label>::zero),
     solutionD_(Vector<label>::zero),
     tetBasePtIsPtr_(NULL),

src/OpenFOAM/meshes/polyMesh/polyPatches/constraint/processor/processorPolyPatch.H

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2012 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -267,6 +267,30 @@ public:
             return neighbProcNo_;
         }
 
+
+        // For testing
+
+            //- Return communicator used for communication
+            label& comm()
+            {
+                return comm_;
+            }
+
+            //- Return processor number
+            int& myProcNo()
+            {
+                return myProcNo_;
+            }
+
+            //- Return neigbour processor number
+            int& neighbProcNo()
+            {
+                return neighbProcNo_;
+            }
+
+
+
+
         //- Does the processor own the patch ?
         virtual bool owner() const
         {

src/Pstream/mpi/UIPread.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2012 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -70,7 +70,7 @@ Foam::UIPstream::UIPstream
 
         label wantedSize = externalBuf_.capacity();
 
-        if (debug)
+        if (debug&1)
         {
             Pout<< "UIPstream::UIPstream : read from:" << fromProcNo
                 << " tag:" << tag << " wanted size:" << wantedSize
@@ -88,7 +88,7 @@ Foam::UIPstream::UIPstream
             externalBuf_.setCapacity(messageSize_);
             wantedSize = messageSize_;
 
-            if (debug)
+            if (debug&1)
             {
                 Pout<< "UIPstream::UIPstream : probed size:" << wantedSize
                     << Foam::endl;
@@ -145,7 +145,7 @@ Foam::UIPstream::UIPstream(const int fromProcNo, PstreamBuffers& buffers)
         // Message is already received into externalBuf
         messageSize_ = buffers.recvBuf_[fromProcNo].size();
 
-        if (debug)
+        if (debug&1)
         {
             Pout<< "UIPstream::UIPstream PstreamBuffers :"
                 << " fromProcNo:" << fromProcNo
@@ -159,7 +159,7 @@ Foam::UIPstream::UIPstream(const int fromProcNo, PstreamBuffers& buffers)
 
         label wantedSize = externalBuf_.capacity();
 
-        if (debug)
+        if (debug&1)
         {
             Pout<< "UIPstream::UIPstream PstreamBuffers :"
                 << " read from:" << fromProcNo
@@ -177,7 +177,7 @@ Foam::UIPstream::UIPstream(const int fromProcNo, PstreamBuffers& buffers)
             externalBuf_.setCapacity(messageSize_);
             wantedSize = messageSize_;
 
-            if (debug)
+            if (debug&1)
             {
                 Pout<< "UIPstream::UIPstream PstreamBuffers : probed size:"
                     << wantedSize << Foam::endl;
@@ -217,7 +217,7 @@ Foam::label Foam::UIPstream::read
     const label communicator
 )
 {
-    if (debug)
+    if (debug&1)
     {
         Pout<< "UIPstream::read : starting read from:" << fromProcNo
             << " tag:" << tag << " comm:" << communicator
@@ -225,6 +225,15 @@ Foam::label Foam::UIPstream::read
             << " commsType:" << UPstream::commsTypeNames[commsType]
             << Foam::endl;
     }
+    if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
+    {
+        Pout<< "UIPstream::read : starting read from:" << fromProcNo
+            << " tag:" << tag << " comm:" << communicator
+            << " wanted size:" << label(bufSize)
+            << " commsType:" << UPstream::commsTypeNames[commsType]
+            << Foam::endl;
+        error::printStack(Pout);
+    }
 
     if (commsType == blocking || commsType == scheduled)
     {
@@ -260,7 +269,7 @@ Foam::label Foam::UIPstream::read
         int messageSize;
         MPI_Get_count(&status, MPI_BYTE, &messageSize);
 
-        if (debug)
+        if (debug&1)
         {
             Pout<< "UIPstream::read : finished read from:" << fromProcNo
                 << " tag:" << tag << " read size:" << label(bufSize)
@@ -310,7 +319,7 @@ Foam::label Foam::UIPstream::read
             return 0;
         }
 
-        if (debug)
+        if (debug&1)
         {
             Pout<< "UIPstream::read : started read from:" << fromProcNo
                 << " tag:" << tag << " read size:" << label(bufSize)

src/Pstream/mpi/UOPwrite.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -43,7 +43,7 @@ bool Foam::UOPstream::write
     const label communicator
 )
 {
-    if (debug)
+    if (debug&1)
     {
         Pout<< "UOPstream::write : starting write to:" << toProcNo
             << " tag:" << tag
@@ -51,6 +51,16 @@ bool Foam::UOPstream::write
             << " commsType:" << UPstream::commsTypeNames[commsType]
             << Foam::endl;
     }
+    if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
+    {
+        Pout<< "UOPstream::write : starting write to:" << toProcNo
+            << " tag:" << tag
+            << " comm:" << communicator << " size:" << label(bufSize)
+            << " commsType:" << UPstream::commsTypeNames[commsType]
+            << Foam::endl;
+        error::printStack(Pout);
+    }
+
 
     PstreamGlobals::checkCommunicator(communicator, toProcNo);
 
@@ -69,7 +79,7 @@ bool Foam::UOPstream::write
             PstreamGlobals::MPICommunicators_[communicator] //MPI_COMM_WORLD
         );
 
-        if (debug)
+        if (debug&1)
         {
             Pout<< "UOPstream::write : finished write to:" << toProcNo
                 << " tag:" << tag << " size:" << label(bufSize)
@@ -89,7 +99,7 @@ bool Foam::UOPstream::write
             PstreamGlobals::MPICommunicators_[communicator] //MPI_COMM_WORLD
         );
 
-        if (debug)
+        if (debug&1)
         {
             Pout<< "UOPstream::write : finished write to:" << toProcNo
                 << " tag:" << tag << " size:" << label(bufSize)
@@ -112,7 +122,7 @@ bool Foam::UOPstream::write
             &request
         );
 
-        if (debug)
+        if (debug&1)
         {
             Pout<< "UOPstream::write : started write to:" << toProcNo
                 << " tag:" << tag << " size:" << label(bufSize)

src/Pstream/mpi/UPstream.C

@@ -68,7 +68,7 @@ bool Foam::UPstream::init(int& argc, char**& argv)
     int myRank;
     MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
 
-    if (debug)
+    if (debug&1)
     {
         Pout<< "UPstream::init : initialised with numProcs:" << numprocs
             << " myRank:" << myRank << endl;
@@ -107,13 +107,12 @@ bool Foam::UPstream::init(int& argc, char**& argv)
     }
 #   endif
 
-    int processorNameLen;
+    //int processorNameLen;
-    char processorName[MPI_MAX_PROCESSOR_NAME];
+    //char processorName[MPI_MAX_PROCESSOR_NAME];
-
+    //
-    MPI_Get_processor_name(processorName, &processorNameLen);
+    //MPI_Get_processor_name(processorName, &processorNameLen);
-    processorName[processorNameLen] = '\0';
+    //processorName[processorNameLen] = '\0';
-
+    //Pout<< "Processor name:" << processorName << endl;
-    Pout<< "Processor name:" << processorName << endl;
 
     return true;
 }
@@ -121,7 +120,7 @@ bool Foam::UPstream::init(int& argc, char**& argv)
 
 void Foam::UPstream::exit(int errnum)
 {
-    if (debug)
+    if (debug&1)
     {
         Pout<< "UPstream::exit." << endl;
     }
@@ -181,6 +180,12 @@ void Foam::reduce
     const label communicator
 )
 {
+    if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
+    {
+        Pout<< "** reducing:" << Value << " with comm:" << communicator
+            << endl;
+        error::printStack(Pout);
+    }
     allReduce(Value, 1, MPI_SCALAR, MPI_SUM, bop, tag, communicator);
 }
 
@@ -193,6 +198,12 @@ void Foam::reduce
     const label communicator
 )
 {
+    if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
+    {
+        Pout<< "** reducing:" << Value << " with comm:" << communicator
+            << endl;
+        error::printStack(Pout);
+    }
     allReduce(Value, 1, MPI_SCALAR, MPI_MIN, bop, tag, communicator);
 }
 
@@ -205,6 +216,12 @@ void Foam::reduce
     const label communicator
 )
 {
+    if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
+    {
+        Pout<< "** reducing:" << Value << " with comm:" << communicator
+            << endl;
+        error::printStack(Pout);
+    }
     allReduce(Value, 2, MPI_SCALAR, MPI_SUM, bop, tag, communicator);
 }
 
@@ -217,6 +234,12 @@ void Foam::sumReduce
     const label communicator
 )
 {
+    if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
+    {
+        Pout<< "** reducing:" << Value << " with comm:" << communicator
+            << endl;
+        error::printStack(Pout);
+    }
     vector2D twoScalars(Value, scalar(Count));
     reduce(twoScalars, sumOp<vector2D>(), tag, communicator);
 
@@ -407,7 +430,7 @@ void Foam::UPstream::resetRequests(const label i)
 
 void Foam::UPstream::waitRequests(const label start)
 {
-    if (debug)
+    if (debug&1)
     {
         Pout<< "UPstream::waitRequests : starting wait for "
             << PstreamGlobals::outstandingRequests_.size()-start
@@ -442,7 +465,7 @@ void Foam::UPstream::waitRequests(const label start)
         resetRequests(start);
     }
 
-    if (debug)
+    if (debug&1)
     {
         Pout<< "UPstream::waitRequests : finished wait." << endl;
     }
@@ -451,7 +474,7 @@ void Foam::UPstream::waitRequests(const label start)
 
 void Foam::UPstream::waitRequest(const label i)
 {
-    if (debug)
+    if (debug&1)
     {
         Pout<< "UPstream::waitRequest : starting wait for request:" << i
             << endl;
@@ -484,7 +507,7 @@ void Foam::UPstream::waitRequest(const label i)
         )   << "MPI_Wait returned with error" << Foam::endl;
     }
 
-    if (debug)
+    if (debug&1)
     {
         Pout<< "UPstream::waitRequest : finished wait for request:" << i
             << endl;
@@ -494,7 +517,7 @@ void Foam::UPstream::waitRequest(const label i)
 
 bool Foam::UPstream::finishedRequest(const label i)
 {
-    if (debug)
+    if (debug&1)
     {
         Pout<< "UPstream::waitRequests : checking finishedRequest:" << i
             << endl;
@@ -520,7 +543,7 @@ bool Foam::UPstream::finishedRequest(const label i)
         MPI_STATUS_IGNORE
     );
 
-    if (debug)
+    if (debug&1)
     {
         Pout<< "UPstream::waitRequests : finished finishedRequest:" << i
             << endl;

src/Pstream/mpi/allReduce.H

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2012 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2012-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -53,7 +53,7 @@ void allReduce
     MPI_Op op,
     const BinaryOp& bop,
     const int tag,
-    const label communicator
+    const int communicator
 );
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //

src/finiteVolume/fvMatrices/fvMatrix/fvMatrix.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2012 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -599,10 +599,28 @@ void Foam::fvMatrix<Type>::relax(const scalar alpha)
             }
         }
 
-        reduce(nNon, sumOp<label>());
+        reduce(nNon, sumOp<label>(), UPstream::msgType(), psi_.mesh().comm());
-        reduce(maxNon, maxOp<scalar>());
+        reduce
-        reduce(sumNon, sumOp<scalar>());
+        (
-        sumNon /= returnReduce(D.size(), sumOp<label>());
+            maxNon,
+            maxOp<scalar>(),
+            UPstream::msgType(),
+            psi_.mesh().comm()
+        );
+        reduce
+        (
+            sumNon,
+            sumOp<scalar>(),
+            UPstream::msgType(),
+            psi_.mesh().comm()
+        );
+        sumNon /= returnReduce
+        (
+            D.size(),
+            sumOp<label>(),
+            UPstream::msgType(),
+            psi_.mesh().comm()
+        );
 
         InfoIn("fvMatrix<Type>::relax(const scalar alpha)")
             << "Matrix dominance test for " << psi_.name() << nl

src/finiteVolume/fvMatrices/fvMatrix/fvMatrixSolve.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2012 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -198,7 +198,10 @@ Foam::solverPerformance Foam::fvMatrix<Type>::solveSegregated
             solverControls
         )->solve(psiCmpt, sourceCmpt, cmpt);
 
-        solverPerf.print(Info);
+        //solverPerf.print(Info);
+Pout<< "** calling solverPerformance::print with comm:" << this->mesh().comm()
+    << endl;
+        solverPerf.print(Info(this->mesh().comm()));
 
         solverPerfVec = max(solverPerfVec, solverPerf);
         solverPerfVec.solverName() = solverPerf.solverName();
@@ -261,7 +264,10 @@ Foam::solverPerformance Foam::fvMatrix<Type>::solveCoupled
         coupledMatrixSolver->solve(psi)
     );
 
-    solverPerf.print(Info);
+    //solverPerf.print(Info);
+Pout<< "** calling solverPerformance::print with comm:" << this->mesh().comm()
+    << endl;
+    solverPerf.print(Info(this->mesh().comm()));
 
     psi.correctBoundaryConditions();
 

src/finiteVolume/fvMesh/fvMesh.H

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2012 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2012-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -270,6 +270,12 @@ public:
                 return boundary().interfaces();
             }
 
+            //- Return communicator used for parallel communication
+            virtual int comm() const
+            {
+                return polyMesh::comm();
+            }
+
             //- Internal face owner
             const labelUList& owner() const
             {