ENH: GAMGInterface: added communicator to arguments.

Extended UIPstream,UOPstream for user defined communicators

src/OpenFOAM/matrices/LUscalarMatrix/LUscalarMatrix.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -33,6 +33,7 @@ License
 
 Foam::LUscalarMatrix::LUscalarMatrix(const scalarSquareMatrix& matrix)
 :
+    comm_(Pstream::worldComm),
     scalarSquareMatrix(matrix),
     pivotIndices_(n())
 {
@@ -46,10 +47,12 @@ Foam::LUscalarMatrix::LUscalarMatrix
     const FieldField<Field, scalar>& interfaceCoeffs,
     const lduInterfaceFieldPtrsList& interfaces
 )
+:
+    comm_(ldum.mesh().comm())
 {
     if (Pstream::parRun())
     {
-        PtrList<procLduMatrix> lduMatrices(Pstream::nProcs());
+        PtrList<procLduMatrix> lduMatrices(Pstream::nProcs(comm_));
 
         label lduMatrixi = 0;
 
@@ -64,25 +67,49 @@ Foam::LUscalarMatrix::LUscalarMatrix
             )
         );
 
-        if (Pstream::master())
+
+Pout<< "LUscalarMatrix :"
+    << " comm:" << comm_
+    << " master:" << Pstream::master(comm_) << endl;
+
+        if (Pstream::master(comm_))
         {
             for
             (
                 int slave=Pstream::firstSlave();
-                slave<=Pstream::lastSlave();
+                slave<=Pstream::lastSlave(comm_);
                 slave++
             )
             {
+Pout<< "Receiving from " << slave
+    << " using comm:" << comm_ << endl;
                 lduMatrices.set
                 (
                     lduMatrixi++,
-                    new procLduMatrix(IPstream(Pstream::scheduled, slave)())
+                    new procLduMatrix
+                    (
+                        IPstream
+                        (
+                            Pstream::scheduled,
+                            slave,
+                            0,          // bufSize
+                            Pstream::msgType(),
+                            comm_
+                        )()
+                    )
                 );
             }
         }
         else
         {
-            OPstream toMaster(Pstream::scheduled, Pstream::masterNo());
+            OPstream toMaster
+            (
+                Pstream::scheduled,
+                Pstream::masterNo(),
+                0,              // bufSize
+                Pstream::msgType(),
+                comm_
+            );
             procLduMatrix cldum
             (
                 ldum,
@@ -93,7 +120,7 @@ Foam::LUscalarMatrix::LUscalarMatrix
 
         }
 
-        if (Pstream::master())
+        if (Pstream::master(comm_))
         {
             label nCells = 0;
             forAll(lduMatrices, i)
@@ -114,7 +141,7 @@ Foam::LUscalarMatrix::LUscalarMatrix
         convert(ldum, interfaceCoeffs, interfaces);
     }
 
-    if (Pstream::master())
+    if (Pstream::master(comm_))
     {
         pivotIndices_.setSize(n());
         LUDecompose(*this, pivotIndices_);

src/OpenFOAM/matrices/LUscalarMatrix/LUscalarMatrix.H

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -58,6 +58,9 @@ class LUscalarMatrix
 {
     // Private data
 
+        //- Communicator to use
+        const label comm_;
+
         //- Processor matrix offsets
         labelList procOffsets_;
 

src/OpenFOAM/matrices/LUscalarMatrix/LUscalarMatrixTemplates.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -34,7 +34,7 @@ void Foam::LUscalarMatrix::solve(Field<Type>& sourceSol) const
     {
         Field<Type> completeSourceSol(n());
 
-        if (Pstream::master())
+        if (Pstream::master(comm_))
         {
             typename Field<Type>::subField
             (
@@ -45,7 +45,7 @@ void Foam::LUscalarMatrix::solve(Field<Type>& sourceSol) const
             for
             (
                 int slave=Pstream::firstSlave();
-                slave<=Pstream::lastSlave();
+                slave<=Pstream::lastSlave(comm_);
                 slave++
             )
             {
@@ -57,7 +57,9 @@ void Foam::LUscalarMatrix::solve(Field<Type>& sourceSol) const
                     (
                         &(completeSourceSol[procOffsets_[slave]])
                     ),
-                    (procOffsets_[slave + 1] - procOffsets_[slave])*sizeof(Type)
+                    (procOffsets_[slave+1]-procOffsets_[slave])*sizeof(Type),
+                    Pstream::msgType(),
+                    comm_
                 );
             }
         }
@@ -68,11 +70,13 @@ void Foam::LUscalarMatrix::solve(Field<Type>& sourceSol) const
                 Pstream::scheduled,
                 Pstream::masterNo(),
                 reinterpret_cast<const char*>(sourceSol.begin()),
-                sourceSol.byteSize()
+                sourceSol.byteSize(),
+                Pstream::msgType(),
+                comm_
             );
         }
 
-        if (Pstream::master())
+        if (Pstream::master(comm_))
         {
             LUBacksubstitute(*this, pivotIndices_, completeSourceSol);
 
@@ -85,7 +89,7 @@ void Foam::LUscalarMatrix::solve(Field<Type>& sourceSol) const
             for
             (
                 int slave=Pstream::firstSlave();
-                slave<=Pstream::lastSlave();
+                slave<=Pstream::lastSlave(comm_);
                 slave++
             )
             {
@@ -97,7 +101,9 @@ void Foam::LUscalarMatrix::solve(Field<Type>& sourceSol) const
                     (
                         &(completeSourceSol[procOffsets_[slave]])
                     ),
-                    (procOffsets_[slave + 1] - procOffsets_[slave])*sizeof(Type)
+                    (procOffsets_[slave + 1]-procOffsets_[slave])*sizeof(Type),
+                    Pstream::msgType(),
+                    comm_
                 );
             }
         }
@@ -108,7 +114,9 @@ void Foam::LUscalarMatrix::solve(Field<Type>& sourceSol) const
                 Pstream::scheduled,
                 Pstream::masterNo(),
                 reinterpret_cast<char*>(sourceSol.begin()),
-                sourceSol.byteSize()
+                sourceSol.byteSize(),
+                Pstream::msgType(),
+                comm_
             );
         }
     }

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C

@@ -240,6 +240,15 @@ void Foam::GAMGAgglomeration::agglomerateLduAddressing
         Pstream::waitRequests();
     }
 
+
+    // Allocate a communicator for the coarse level
+    label coarseComm = UPstream::allocateCommunicator
+    (
+        fineMesh.comm(),
+        identity(UPstream::nProcs(fineMesh.comm()))  //TBD
+    );
+
+
     // Add the coarse level
     forAll(fineInterfaces, inti)
     {
@@ -259,7 +268,8 @@ void Foam::GAMGAgglomeration::agglomerateLduAddressing
                         Pstream::nonBlocking,
                         restrictMap
                     ),
-                    fineLevelIndex
+                    fineLevelIndex,
+                    coarseComm
                 ).ptr()
             );
 
@@ -280,7 +290,7 @@ void Foam::GAMGAgglomeration::agglomerateLduAddressing
             coarseInterfaceAddr,
             coarseInterfaces,
             fineMeshAddr.patchSchedule(),
-            fineMesh.comm(),
+            coarseComm,
             true
         )
     );

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomeration.C

@@ -196,6 +196,16 @@ const Foam::GAMGAgglomeration& Foam::GAMGAgglomeration::New
 
 Foam::GAMGAgglomeration::~GAMGAgglomeration()
 {
+    // Temporary store the user-defined communicators so we can delete them
+    labelList communicators(meshLevels_.size());
+    forAll(meshLevels_, leveli)
+    {
+        communicators[leveli] = meshLevels_[leveli].comm();
+    }
+
+    Pout<< "~GAMGAgglomeration() : current communicators:" << communicators
+        << endl;
+
     // Clear the interface storage by hand.
     // It is a list of ptrs not a PtrList for consistency of the interface
     for (label leveli=1; leveli<interfaceLevels_.size(); leveli++)
@@ -210,6 +220,11 @@ Foam::GAMGAgglomeration::~GAMGAgglomeration()
             }
         }
     }
+
+    forAll(communicators, i)
+    {
+        UPstream::freeCommunicator(communicators[i]);
+    }
 }
 
 

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolver.C

@@ -94,6 +94,10 @@ Foam::GAMGSolver::GAMGSolver
 
         if (directSolveCoarsest_)
         {
+            label coarseComm = matrixLevels_[coarsestLevel].mesh().comm();
+            label oldWarn = UPstream::warnComm;
+            UPstream::warnComm = coarseComm;
+
             coarsestLUMatrixPtr_.set
             (
                 new LUscalarMatrix
@@ -103,6 +107,8 @@ Foam::GAMGSolver::GAMGSolver
                     interfaceLevels_[coarsestLevel]
                 )
             );
+
+            UPstream::warnComm = oldWarn;
         }
     }
     else

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverSolve.C

@@ -441,6 +441,11 @@ void Foam::GAMGSolver::solveCoarsestLevel
     const scalarField& coarsestSource
 ) const
 {
+    const label coarsestLevel = matrixLevels_.size() - 1;
+    label coarseComm = matrixLevels_[coarsestLevel].mesh().comm();
+    label oldWarn = UPstream::warnComm;
+    UPstream::warnComm = coarseComm;
+
     if (directSolveCoarsest_)
     {
         coarsestCorrField = coarsestSource;
@@ -489,9 +494,11 @@ void Foam::GAMGSolver::solveCoarsestLevel
 
         if (debug >= 2)
         {
-            coarseSolverPerf.print(Info(matrix().mesh().comm()));
+            coarseSolverPerf.print(Info(coarseComm));
         }
     }
+
+    UPstream::warnComm = oldWarn;
 }
 
 

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaceFields/processorGAMGInterfaceField/processorGAMGInterfaceField.C

@@ -79,6 +79,9 @@ void Foam::processorGAMGInterfaceField::initInterfaceMatrixUpdate
     const Pstream::commsTypes commsType
 ) const
 {
+label oldWarn = UPstream::warnComm;
+UPstream::warnComm = comm();
+
     procInterface_.interfaceInternalField(psiInternal, scalarSendBuf_);
 
     if (commsType == Pstream::nonBlocking && !Pstream::floatTransfer)
@@ -93,7 +96,7 @@ void Foam::processorGAMGInterfaceField::initInterfaceMatrixUpdate
             reinterpret_cast<char*>(scalarReceiveBuf_.begin()),
             scalarReceiveBuf_.byteSize(),
             procInterface_.tag(),
-            procInterface_.comm()
+            comm()
         );
 
         outstandingSendRequest_ = UPstream::nRequests();
@@ -104,7 +107,7 @@ void Foam::processorGAMGInterfaceField::initInterfaceMatrixUpdate
             reinterpret_cast<const char*>(scalarSendBuf_.begin()),
             scalarSendBuf_.byteSize(),
             procInterface_.tag(),
-            procInterface_.comm()
+            comm()
         );
     }
     else
@@ -113,6 +116,8 @@ void Foam::processorGAMGInterfaceField::initInterfaceMatrixUpdate
     }
 
     const_cast<processorGAMGInterfaceField&>(*this).updatedMatrix() = false;
+
+UPstream::warnComm = oldWarn;
 }
 
 
@@ -130,6 +135,9 @@ void Foam::processorGAMGInterfaceField::updateInterfaceMatrix
         return;
     }
 
+label oldWarn = UPstream::warnComm;
+UPstream::warnComm = comm();
+
     const labelUList& faceCells = procInterface_.faceCells();
 
     if (commsType == Pstream::nonBlocking && !Pstream::floatTransfer)
@@ -173,6 +181,8 @@ void Foam::processorGAMGInterfaceField::updateInterfaceMatrix
     }
 
     const_cast<processorGAMGInterfaceField&>(*this).updatedMatrix() = true;
+
+UPstream::warnComm = oldWarn;
 }
 
 

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaces/GAMGInterface/GAMGInterface.H

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2012 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2012-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -101,7 +101,8 @@ public:
                 const lduInterface& fineInterface,
                 const labelField& localRestrictAddressing,
                 const labelField& neighbourRestrictAddressing,
-                const label fineLevelIndex
+                const label fineLevelIndex,
+                const label coarseComm
             ),
             (
                 index,
@@ -109,7 +110,8 @@ public:
                 fineInterface,
                 localRestrictAddressing,
                 neighbourRestrictAddressing,
-                fineLevelIndex
+                fineLevelIndex,
+                coarseComm
             )
         );
 
@@ -125,7 +127,8 @@ public:
             const lduInterface& fineInterface,
             const labelField& localRestrictAddressing,
             const labelField& neighbourRestrictAddressing,
-            const label fineLevelIndex
+            const label fineLevelIndex,
+            const label coarseComm
         );
 
 
@@ -140,7 +143,8 @@ public:
             const lduInterface&,
             const labelField&,
             const labelField&,
-            const label fineLevelIndex = -1
+            const label fineLevelIndex = -1,
+            const label coarseComm = UPstream::worldComm
         )
         :
             index_(index),

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaces/GAMGInterface/GAMGInterfaceNew.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -37,7 +37,8 @@ Foam::autoPtr<Foam::GAMGInterface> Foam::GAMGInterface::New
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
     const labelField& neighbourRestrictAddressing,
-    const label fineLevelIndex
+    const label fineLevelIndex,
+    const label coarseComm
 )
 {
     const word coupleType(fineInterface.type());
@@ -68,7 +69,8 @@ Foam::autoPtr<Foam::GAMGInterface> Foam::GAMGInterface::New
             fineInterface,
             localRestrictAddressing,
             neighbourRestrictAddressing,
-            fineLevelIndex
+            fineLevelIndex,
+            coarseComm
         )
     );
 }

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaces/cyclicGAMGInterface/cyclicGAMGInterface.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -51,7 +51,8 @@ Foam::cyclicGAMGInterface::cyclicGAMGInterface
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
     const labelField& neighbourRestrictAddressing,
-    const label fineLevelIndex
+    const label fineLevelIndex,
+    const label coarseComm
 )
 :
     GAMGInterface

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaces/cyclicGAMGInterface/cyclicGAMGInterface.H

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -85,7 +85,8 @@ public:
             const lduInterface& fineInterface,
             const labelField& restrictAddressing,
             const labelField& neighbourRestrictAddressing,
-            const label fineLevelIndex
+            const label fineLevelIndex,
+            const label coarseComm
         );
 
 

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaces/processorCyclicGAMGInterface/processorCyclicGAMGInterface.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -50,7 +50,8 @@ Foam::processorCyclicGAMGInterface::processorCyclicGAMGInterface
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
     const labelField& neighbourRestrictAddressing,
-    const label fineLevelIndex
+    const label fineLevelIndex,
+    const label coarseComm
 )
 :
     processorGAMGInterface
@@ -60,7 +61,8 @@ Foam::processorCyclicGAMGInterface::processorCyclicGAMGInterface
         fineInterface,
         localRestrictAddressing,
         neighbourRestrictAddressing,
-        fineLevelIndex
+        fineLevelIndex,
+        coarseComm
     )
 {}
 

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaces/processorCyclicGAMGInterface/processorCyclicGAMGInterface.H

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -76,7 +76,8 @@ public:
             const lduInterface& fineInterface,
             const labelField& restrictAddressing,
             const labelField& neighbourRestrictAddressing,
-            const label fineLevelIndex
+            const label fineLevelIndex,
+            const label coarseComm
         );
 
 

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaces/processorGAMGInterface/processorGAMGInterface.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -51,7 +51,8 @@ Foam::processorGAMGInterface::processorGAMGInterface
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
     const labelField& neighbourRestrictAddressing,
-    const label fineLevelIndex
+    const label fineLevelIndex,
+    const label coarseComm
 )
 :
     GAMGInterface
@@ -62,7 +63,8 @@ Foam::processorGAMGInterface::processorGAMGInterface
         localRestrictAddressing,
         neighbourRestrictAddressing
     ),
-    fineProcInterface_(refCast<const processorLduInterface>(fineInterface))
+    fineProcInterface_(refCast<const processorLduInterface>(fineInterface)),
+    comm_(coarseComm)
 {
     // From coarse face to coarse cell
     DynamicList<label> dynFaceCells(localRestrictAddressing.size());
@@ -140,7 +142,12 @@ void Foam::processorGAMGInterface::initInternalFieldTransfer
     const labelUList& iF
 ) const
 {
+label oldWarn = UPstream::warnComm;
+UPstream::warnComm = comm();
+
     send(commsType, interfaceInternalField(iF)());
+
+UPstream::warnComm = oldWarn;
 }
 
 
@@ -150,7 +157,13 @@ Foam::tmp<Foam::labelField> Foam::processorGAMGInterface::internalFieldTransfer
     const labelUList& iF
 ) const
 {
-    return receive<label>(commsType, this->size());
+label oldWarn = UPstream::warnComm;
+UPstream::warnComm = comm();
+
+    tmp<Field<label> > tfld(receive<label>(commsType, this->size()));
+
+UPstream::warnComm = oldWarn;
+    return tfld;
 }
 
 

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaces/processorGAMGInterface/processorGAMGInterface.H

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -59,6 +59,9 @@ class processorGAMGInterface
         //  agglomerated
         const processorLduInterface& fineProcInterface_;
 
+        //- Communicator to use for parallel communication
+        const label comm_;
+
 
     // Private Member Functions
 
@@ -85,7 +88,8 @@ public:
             const lduInterface& fineInterface,
             const labelField& restrictAddressing,
             const labelField& neighbourRestrictAddressing,
-            const label fineLevelIndex
+            const label fineLevelIndex,
+            const label coarseComm
         );
 
 
@@ -141,7 +145,7 @@ public:
             //- Return communicator used for sending
             virtual int comm() const
             {
-                return fineProcInterface_.comm();
+                return comm_;
             }
 };
 

src/OpenFOAM/meshes/lduMesh/lduPrimitiveMesh.H

@@ -34,6 +34,7 @@ Description
 
 #include "lduMesh.H"
 #include "labelList.H"
+#include "processorLduInterface.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
@@ -112,6 +113,24 @@ public:
                 << " interfaces:" << interfaces_.size()
                 << " comm:" << comm_
                 << endl;
+            forAll(interfaces_, i)
+            {
+                if (interfaces_.set(i))
+                {
+                    if (isA<processorLduInterface>(interfaces_[i]))
+                    {
+                        const processorLduInterface& pi = refCast
+                        <
+                            const processorLduInterface
+                        >(interfaces_[i]);
+
+                        Pout<< "    comm:" << pi.comm()
+                            << " myProcNo:" << pi.myProcNo()
+                            << " neighbProcNo:" << pi.neighbProcNo()
+                            << endl;
+                    }
+                }
+            }
         }
 
 
@@ -144,6 +163,24 @@ public:
                 << " interfaces:" << interfaces_.size()
                 << " comm:" << comm_
                 << endl;
+            forAll(interfaces_, i)
+            {
+                if (interfaces_.set(i))
+                {
+                    if (isA<processorLduInterface>(interfaces_[i]))
+                    {
+                        const processorLduInterface& pi = refCast
+                        <
+                            const processorLduInterface
+                        >(interfaces_[i]);
+
+                        Pout<< "    comm:" << pi.comm()
+                            << " myProcNo:" << pi.myProcNo()
+                            << " neighbProcNo:" << pi.neighbProcNo()
+                            << endl;
+                    }
+                }
+            }
         }
 
 

src/Pstream/mpi/UIPread.C

@@ -82,7 +82,13 @@ Foam::UIPstream::UIPstream
         // and set it
         if (!wantedSize)
         {
-            MPI_Probe(fromProcNo_, tag_, MPI_COMM_WORLD, &status);
+            MPI_Probe
+            (
+                fromProcNo_,
+                tag_,
+                PstreamGlobals::MPICommunicators_[comm_],
+                &status
+            );
             MPI_Get_count(&status, MPI_BYTE, &messageSize_);
 
             externalBuf_.setCapacity(messageSize_);
@@ -171,7 +177,13 @@ Foam::UIPstream::UIPstream(const int fromProcNo, PstreamBuffers& buffers)
         // and set it
         if (!wantedSize)
         {
-            MPI_Probe(fromProcNo_, tag_, MPI_COMM_WORLD, &status);
+            MPI_Probe
+            (
+                fromProcNo_,
+                tag_,
+                PstreamGlobals::MPICommunicators_[comm_],
+                &status
+            );
             MPI_Get_count(&status, MPI_BYTE, &messageSize_);
 
             externalBuf_.setCapacity(messageSize_);
@@ -231,6 +243,7 @@ Foam::label Foam::UIPstream::read
             << " tag:" << tag << " comm:" << communicator
             << " wanted size:" << label(bufSize)
             << " commsType:" << UPstream::commsTypeNames[commsType]
+            << " warnComm:" << UPstream::warnComm
             << Foam::endl;
         error::printStack(Pout);
     }

src/Pstream/mpi/UOPwrite.C

@@ -57,6 +57,7 @@ bool Foam::UOPstream::write
             << " tag:" << tag
             << " comm:" << communicator << " size:" << label(bufSize)
             << " commsType:" << UPstream::commsTypeNames[commsType]
+            << " warnComm:" << UPstream::warnComm
             << Foam::endl;
         error::printStack(Pout);
     }

src/Pstream/mpi/UPstream.C

@@ -183,6 +183,7 @@ void Foam::reduce
     if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
     {
         Pout<< "** reducing:" << Value << " with comm:" << communicator
+            << " warnComm:" << UPstream::warnComm
             << endl;
         error::printStack(Pout);
     }
@@ -201,6 +202,7 @@ void Foam::reduce
     if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
     {
         Pout<< "** reducing:" << Value << " with comm:" << communicator
+            << " warnComm:" << UPstream::warnComm
             << endl;
         error::printStack(Pout);
     }
@@ -219,6 +221,7 @@ void Foam::reduce
     if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
     {
         Pout<< "** reducing:" << Value << " with comm:" << communicator
+            << " warnComm:" << UPstream::warnComm
             << endl;
         error::printStack(Pout);
     }
@@ -237,6 +240,7 @@ void Foam::sumReduce
     if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
     {
         Pout<< "** reducing:" << Value << " with comm:" << communicator
+            << " warnComm:" << UPstream::warnComm
             << endl;
         error::printStack(Pout);
     }

src/meshTools/AMIInterpolation/GAMG/interfaces/cyclicAMIGAMGInterface/cyclicAMIGAMGInterface.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -51,7 +51,8 @@ Foam::cyclicAMIGAMGInterface::cyclicAMIGAMGInterface
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
     const labelField& neighbourRestrictAddressing,
-    const label fineLevelIndex
+    const label fineLevelIndex,
+    const label coarseComm
 )
 :
     GAMGInterface

src/meshTools/AMIInterpolation/GAMG/interfaces/cyclicAMIGAMGInterface/cyclicAMIGAMGInterface.H

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -88,7 +88,8 @@ public:
             const lduInterface& fineInterface,
             const labelField& restrictAddressing,
             const labelField& neighbourRestrictAddressing,
-            const label fineLevelIndex
+            const label fineLevelIndex,
+            const label coarseComm
         );
 
 

src/meshTools/regionCoupled/GAMG/interfaces/regionCoupledGAMGInterface/regionCoupledBaseGAMGInterface.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -38,7 +38,8 @@ Foam::regionCoupledBaseGAMGInterface::regionCoupledBaseGAMGInterface
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
     const labelField& neighbourRestrictAddressing,
-    const label fineLevelIndex
+    const label fineLevelIndex,
+    const label coarseComm
 )
 :
     GAMGInterface
@@ -48,7 +49,8 @@ Foam::regionCoupledBaseGAMGInterface::regionCoupledBaseGAMGInterface
         fineInterface,
         localRestrictAddressing,
         neighbourRestrictAddressing,
-        fineLevelIndex
+        fineLevelIndex,
+        coarseComm
     ),
     fineRegionCoupledLduInterface_
     (

src/meshTools/regionCoupled/GAMG/interfaces/regionCoupledGAMGInterface/regionCoupledBaseGAMGInterface.H

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -86,7 +86,8 @@ public:
             const lduInterface& fineInterface,
             const labelField& restrictAddressing,
             const labelField& neighbourRestrictAddressing,
-            const label fineLevelIndex
+            const label fineLevelIndex,
+            const label coarseComm
         );
 
 

src/meshTools/regionCoupled/GAMG/interfaces/regionCoupledGAMGInterface/regionCoupledGAMGInterface.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -49,7 +49,8 @@ Foam::regionCoupledGAMGInterface::regionCoupledGAMGInterface
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
     const labelField& neighbourRestrictAddressing,
-    const label fineLevelIndex
+    const label fineLevelIndex,
+    const label coarseComm
 )
 :
     regionCoupledBaseGAMGInterface
@@ -59,7 +60,8 @@ Foam::regionCoupledGAMGInterface::regionCoupledGAMGInterface
         fineInterface,
         localRestrictAddressing,
         neighbourRestrictAddressing,
-        fineLevelIndex
+        fineLevelIndex,
+        coarseComm
     )
 {}
 

src/meshTools/regionCoupled/GAMG/interfaces/regionCoupledGAMGInterface/regionCoupledGAMGInterface.H

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -76,7 +76,8 @@ public:
             const lduInterface& fineInterface,
             const labelField& restrictAddressing,
             const labelField& neighbourRestrictAddressing,
-            const label fineLevelIndex
+            const label fineLevelIndex,
+            const label coarseComm
         );
 
 

src/meshTools/regionCoupled/GAMG/interfaces/regionCoupledGAMGInterface/regionCoupledWallGAMGInterface.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -49,7 +49,8 @@ Foam::regionCoupledWallGAMGInterface::regionCoupledWallGAMGInterface
     const lduInterface& fineInterface,
     const labelField& localRestrictAddressing,
     const labelField& neighbourRestrictAddressing,
-    const label fineLevelIndex
+    const label fineLevelIndex,
+    const label coarseComm
 )
 :
     regionCoupledBaseGAMGInterface
@@ -59,7 +60,8 @@ Foam::regionCoupledWallGAMGInterface::regionCoupledWallGAMGInterface
         fineInterface,
         localRestrictAddressing,
         neighbourRestrictAddressing,
-        fineLevelIndex
+        fineLevelIndex,
+        coarseComm
     )
 {}
 

src/meshTools/regionCoupled/GAMG/interfaces/regionCoupledGAMGInterface/regionCoupledWallGAMGInterface.H

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -78,7 +78,8 @@ public:
             const lduInterface& fineInterface,
             const labelField& restrictAddressing,
             const labelField& neighbourRestrictAddressing,
-            const label fineLevelIndex
+            const label fineLevelIndex,
+            const label coarseComm
         );