From 314d05cfdb714379f6eba1ac8bda2de70abc3958 Mon Sep 17 00:00:00 2001
From: andy <andy>
Date: Fri, 10 May 2013 10:15:31 +0100
Subject: [PATCH 1/3] STYLE: Minor typo corrections

---
 .../icoUncoupledKinematicParcelFoam.C                         | 4 ++--
 .../uncoupledKinematicParcelFoam.C                            | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/applications/solvers/lagrangian/icoUncoupledKinematicParcelFoam/icoUncoupledKinematicParcelFoam.C b/applications/solvers/lagrangian/icoUncoupledKinematicParcelFoam/icoUncoupledKinematicParcelFoam.C
index 6e9c2bfe7e..2bf1238452 100644
--- a/applications/solvers/lagrangian/icoUncoupledKinematicParcelFoam/icoUncoupledKinematicParcelFoam.C
+++ b/applications/solvers/lagrangian/icoUncoupledKinematicParcelFoam/icoUncoupledKinematicParcelFoam.C
@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2012 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -22,7 +22,7 @@ License
     along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
 
 Application
-    uncoupledKinematicParcelFoam
+    icoUncoupledKinematicParcelFoam
 
 Description
     Transient solver for the passive transport of a single kinematic
diff --git a/applications/solvers/lagrangian/uncoupledKinematicParcelFoam/uncoupledKinematicParcelFoam.C b/applications/solvers/lagrangian/uncoupledKinematicParcelFoam/uncoupledKinematicParcelFoam.C
index 661353669b..352678f853 100644
--- a/applications/solvers/lagrangian/uncoupledKinematicParcelFoam/uncoupledKinematicParcelFoam.C
+++ b/applications/solvers/lagrangian/uncoupledKinematicParcelFoam/uncoupledKinematicParcelFoam.C
@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2012 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -26,7 +26,7 @@ Application
 
 Description
     Transient solver for the passive transport of a single kinematic
-    particle could.
+    particle cloud.
 
     Uses a pre- calculated velocity field to evolve the cloud.
 

From 2f1b6d9e96a13bbafb79cadd72fa8eb28c324697 Mon Sep 17 00:00:00 2001
From: mattijs <mattijs>
Date: Fri, 10 May 2013 13:13:33 +0100
Subject: [PATCH 2/3] ENH: GAMGProcAgglomeration: use non-blocking transfers

---
 .../GAMGAgglomerateLduAddressing.C            |   5 +-
 .../GAMGAgglomerationTemplates.C              |  21 +-
 .../manualGAMGProcAgglomeration.C             |   6 +-
 .../manualGAMGProcAgglomeration.H             |  14 +-
 .../lduMatrix/solvers/GAMG/GAMGSolverSolve.C  |  22 --
 .../polyMesh/globalMeshData/globalIndex.H     |  24 +-
 .../globalMeshData/globalIndexTemplates.C     | 354 +++++++++++-------
 7 files changed, 266 insertions(+), 180 deletions(-)

diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C
index 511aeb70e2..e6781ed23f 100644
--- a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C
+++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C
@@ -488,7 +488,10 @@ void Foam::GAMGAgglomeration::procAgglomerateRestrictAddressing
         comm,
         procIDs,
         restrictAddressing_[levelIndex],
-        procRestrictAddressing
+        procRestrictAddressing,
+
+        UPstream::msgType(),
+        Pstream::nonBlocking    //Pstream::scheduled
     );
 
 
diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomerationTemplates.C b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomerationTemplates.C
index ce4fa0073a..39d69458e2 100644
--- a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomerationTemplates.C
+++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomerationTemplates.C
@@ -126,7 +126,15 @@ void Foam::GAMGAgglomeration::restrictField
         const List<int>& procIDs = agglomProcIDs(coarseLevelIndex);
         const labelList& offsets = cellOffsets(coarseLevelIndex);
 
-        globalIndex::gather(offsets, fineComm, procIDs, cf);
+        globalIndex::gather
+        (
+            offsets,
+            fineComm,
+            procIDs,
+            cf,
+            UPstream::msgType(),
+            Pstream::nonBlocking    //Pstream::scheduled
+        );
     }
 }
 
@@ -194,7 +202,16 @@ void Foam::GAMGAgglomeration::prolongField
         label localSize = nCells_[levelIndex];
 
         Field<Type> allCf(localSize);
-        globalIndex::scatter(offsets, coarseComm, procIDs, cf, allCf);
+        globalIndex::scatter
+        (
+            offsets,
+            coarseComm,
+            procIDs,
+            cf,
+            allCf,
+            UPstream::msgType(),
+            Pstream::nonBlocking    //Pstream::scheduled
+        );
 
         forAll(fineToCoarse, i)
         {
diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/manualGAMGProcAgglomeration/manualGAMGProcAgglomeration.C b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/manualGAMGProcAgglomeration/manualGAMGProcAgglomeration.C
index 1fea00057c..c67ce2b18b 100644
--- a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/manualGAMGProcAgglomeration/manualGAMGProcAgglomeration.C
+++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/manualGAMGProcAgglomeration/manualGAMGProcAgglomeration.C
@@ -109,7 +109,7 @@ bool Foam::manualGAMGProcAgglomeration::agglomerate()
                     // My processor id
                     const label myProcID = Pstream::myProcNo(levelMesh.comm());
 
-                    const List<clusterAndMaster>& clusters =
+                    const List<labelList>& clusters =
                         procAgglomMaps_[i].second();
 
                     // Coarse to fine master processor
@@ -125,8 +125,8 @@ bool Foam::manualGAMGProcAgglomeration::agglomerate()
 
                     forAll(clusters, coarseI)
                     {
-                        const labelList& cluster = clusters[coarseI].first();
-                        coarseToMaster[coarseI] = clusters[coarseI].second();
+                        const labelList& cluster = clusters[coarseI];
+                        coarseToMaster[coarseI] = cluster[0];
 
                         forAll(cluster, i)
                         {
diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/manualGAMGProcAgglomeration/manualGAMGProcAgglomeration.H b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/manualGAMGProcAgglomeration/manualGAMGProcAgglomeration.H
index c7efec92f0..1c93711dd3 100644
--- a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/manualGAMGProcAgglomeration/manualGAMGProcAgglomeration.H
+++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/manualGAMGProcAgglomeration/manualGAMGProcAgglomeration.H
@@ -30,19 +30,23 @@ Description
     In the GAMG control dictionary:
 
         processorAgglomerator manual;
+        // List of level+procagglomeration where
+        // procagglomeration is a set of labelLists. Each labelList is
+        // a cluster of processor which gets combined onto the first element
+        // in the list.
         processorAgglomeration
         (
             (
                 3           //at level 3
                 (
-                    ((0 1) 0)   //coarse 0 from 0,1 (and moved onto 0)
-                    ((2 3) 3)   //coarse 1 from 2,3 (and moved onto 3)
+                    (0 1)   //coarse 0 from 0,1 (and moved onto 0)
+                    (3 2)   //coarse 1 from 2,3 (and moved onto 3)
                 )
             )
             (
                 6           //at level6
                 (
-                    ((0 1) 0)   //coarse 0 from 0,1 (and moved onto 0)
+                    (0 1)   //coarse 0 from 0,1 (and moved onto 0)
                 )
             )
         );
@@ -76,10 +80,8 @@ class manualGAMGProcAgglomeration
 {
     // Private data
 
-        typedef Tuple2<labelList, label> clusterAndMaster;
-
         //- Per level the agglomeration map
-        const List<Tuple2<label, List<clusterAndMaster> > > procAgglomMaps_;
+        const List<Tuple2<label, List<labelList> > > procAgglomMaps_;
 
         //- Any allocated communicators
         DynamicList<label> comms_;
diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverSolve.C b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverSolve.C
index ae2cfa21ad..9677a17a9b 100644
--- a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverSolve.C
+++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverSolve.C
@@ -114,13 +114,6 @@ Foam::solverPerformance Foam::GAMGSolver::solve
                 cmpt
             );
 
-
-            //Pout<< "finestCorrection:" << finestCorrection << endl;
-            //Pout<< "finestResidual:" << finestResidual << endl;
-            //Pout<< "psi:" << psi << endl;
-            //Pout<< "Apsi:" << Apsi << endl;
-
-
             // Calculate finest level residual field
             matrix_.Amul(Apsi, psi, interfaceBouCoeffs_, interfaces_, cmpt);
             finestResidual = source;
@@ -205,7 +198,6 @@ void Foam::GAMGSolver::Vcycle
                     scratch1,
                     coarseCorrFields[leveli].size()
                 );
-                //scalarField ACf(coarseCorrFields[leveli].size(), VGREAT);
 
                 // Scale coarse-grid correction field
                 // but not on the coarsest level because it evaluates to 1
@@ -218,8 +210,6 @@ void Foam::GAMGSolver::Vcycle
                         (
                             ACf.operator const scalarField&()
                         ),
-                        //ACf,
-
                         matrixLevels_[leveli],
                         interfaceLevelsBouCoeffs_[leveli],
                         interfaceLevels_[leveli],
@@ -235,7 +225,6 @@ void Foam::GAMGSolver::Vcycle
                     (
                         ACf.operator const scalarField&()
                     ),
-                    //ACf,
                     coarseCorrFields[leveli],
                     interfaceLevelsBouCoeffs_[leveli],
                     interfaceLevels_[leveli],
@@ -294,11 +283,6 @@ void Foam::GAMGSolver::Vcycle
                 scratch2,
                 coarseCorrFields[leveli].size()
             );
-            //scalarField preSmoothedCoarseCorrField
-            //(
-            //    coarseCorrFields[leveli].size(),
-            //    VGREAT
-            //);
 
             // Only store the preSmoothedCoarseCorrField if pre-smoothing is
             // used
@@ -328,12 +312,6 @@ void Foam::GAMGSolver::Vcycle
             );
             scalarField& ACfRef =
                 const_cast<scalarField&>(ACf.operator const scalarField&());
-            //scalarField ACfRef
-            //(
-            //    coarseCorrFields[leveli].size(),
-            //    VGREAT
-            //);
-
 
             if (interpolateCorrection_)
             {
diff --git a/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalIndex.H b/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalIndex.H
index 80263b91d3..1aa39c1754 100644
--- a/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalIndex.H
+++ b/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalIndex.H
@@ -160,7 +160,8 @@ public:
                 const labelList& procIDs,
                 const UList<Type>& fld,
                 List<Type>& allFld,
-                const int tag = UPstream::msgType()
+                const int tag = UPstream::msgType(),
+                const Pstream::commsTypes commsType=Pstream::nonBlocking
             );
 
             //- Collect data in processor order on master (== procIDs[0]).
@@ -172,10 +173,11 @@ public:
                 const labelList& procIDs,
                 const UList<Type>& fld,
                 List<Type>& allFld,
-                const int tag = UPstream::msgType()
+                const int tag = UPstream::msgType(),
+                const Pstream::commsTypes commsType=Pstream::nonBlocking
             ) const
             {
-                gather(offsets_, comm, procIDs, fld, allFld, tag);
+                gather(offsets_, comm, procIDs, fld, allFld, tag, commsType);
             }
 
             //- Inplace collect data in processor order on master
@@ -187,7 +189,8 @@ public:
                 const label comm,
                 const labelList& procIDs,
                 List<Type>& fld,
-                const int tag = UPstream::msgType()
+                const int tag = UPstream::msgType(),
+                const Pstream::commsTypes commsType=Pstream::nonBlocking
             );
 
             //- Inplace collect data in processor order on master
@@ -198,10 +201,11 @@ public:
                 const label comm,
                 const labelList& procIDs,
                 List<Type>& fld,
-                const int tag = UPstream::msgType()
+                const int tag = UPstream::msgType(),
+                const Pstream::commsTypes commsType=Pstream::nonBlocking
             ) const
             {
-                gather(offsets_, comm, procIDs, fld, tag);
+                gather(offsets_, comm, procIDs, fld, tag, commsType);
             }
 
             //- Distribute data in processor order. Requires fld to be sized!
@@ -213,7 +217,8 @@ public:
                 const labelList& procIDs,
                 const UList<Type>& allFld,
                 UList<Type>& fld,
-                const int tag = UPstream::msgType()
+                const int tag = UPstream::msgType(),
+                const Pstream::commsTypes commsType=Pstream::nonBlocking
             );
 
             //- Distribute data in processor order. Requires fld to be sized!
@@ -224,10 +229,11 @@ public:
                 const labelList& procIDs,
                 const UList<Type>& allFld,
                 UList<Type>& fld,
-                const int tag = UPstream::msgType()
+                const int tag = UPstream::msgType(),
+                const Pstream::commsTypes commsType=Pstream::nonBlocking
             ) const
             {
-                scatter(offsets_, comm, procIDs, allFld, fld, tag);
+                scatter(offsets_, comm, procIDs, allFld, fld, tag, commsType);
             }
 
 
diff --git a/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalIndexTemplates.C b/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalIndexTemplates.C
index 73d2ea25a8..08435991ee 100644
--- a/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalIndexTemplates.C
+++ b/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalIndexTemplates.C
@@ -30,30 +30,75 @@ License
 template<class Type>
 void Foam::globalIndex::gather
 (
-    const labelUList& offsets,
+    const labelUList& off,
     const label comm,
     const labelList& procIDs,
     const UList<Type>& fld,
     List<Type>& allFld,
-    const int tag
+    const int tag,
+    const Pstream::commsTypes commsType
 )
 {
     if (Pstream::myProcNo(comm) == procIDs[0])
     {
-        allFld.setSize(offsets.last());
+        allFld.setSize(off.last());
 
         // Assign my local data
         SubList<Type>(allFld, fld.size(), 0).assign(fld);
 
-        for (label i = 1; i < procIDs.size(); i++)
+        if (commsType == Pstream::scheduled || commsType == Pstream::blocking)
         {
-            SubList<Type> procSlot(allFld, offsets[i+1]-offsets[i], offsets[i]);
-
-            if (contiguous<Type>())
+            for (label i = 1; i < procIDs.size(); i++)
             {
+                SubList<Type> procSlot(allFld, off[i+1]-off[i], off[i]);
+
+                if (contiguous<Type>())
+                {
+                    IPstream::read
+                    (
+                        commsType,
+                        procIDs[i],
+                        reinterpret_cast<char*>(procSlot.begin()),
+                        procSlot.byteSize(),
+                        tag,
+                        comm
+                    );
+                }
+                else
+                {
+                    IPstream fromSlave
+                    (
+                        commsType,
+                        procIDs[i],
+                        0,
+                        tag,
+                        comm
+                    );
+                    fromSlave >> procSlot;
+                }
+            }
+        }
+        else
+        {
+            // nonBlocking
+
+            if (!contiguous<Type>())
+            {
+                FatalErrorIn("globalIndex::gather(..)")
+                    << "nonBlocking not supported for non-contiguous data"
+                    << exit(FatalError);
+            }
+
+            label startOfRequests = Pstream::nRequests();
+
+            // Set up reads
+            for (label i = 1; i < procIDs.size(); i++)
+            {
+                SubList<Type> procSlot(allFld, off[i+1]-off[i], off[i]);
+
                 IPstream::read
                 (
-                    Pstream::scheduled,
+                    commsType,
                     procIDs[i],
                     reinterpret_cast<char*>(procSlot.begin()),
                     procSlot.byteSize(),
@@ -61,45 +106,66 @@ void Foam::globalIndex::gather
                     comm
                 );
             }
-            else
-            {
-                IPstream fromSlave
-                (
-                    Pstream::scheduled,
-                    procIDs[i],
-                    0,
-                    tag,
-                    comm
-                );
-                fromSlave >> procSlot;
-            }
+
+            // Wait for all to finish
+            Pstream::waitRequests(startOfRequests);
         }
     }
     else
     {
-        if (contiguous<Type>())
+        if (commsType == Pstream::scheduled || commsType == Pstream::blocking)
         {
+            if (contiguous<Type>())
+            {
+                OPstream::write
+                (
+                    commsType,
+                    procIDs[0],
+                    reinterpret_cast<const char*>(fld.begin()),
+                    fld.byteSize(),
+                    tag,
+                    comm
+                );
+            }
+            else
+            {
+                OPstream toMaster
+                (
+                    commsType,
+                    procIDs[0],
+                    0,
+                    tag,
+                    comm
+                );
+                toMaster << fld;
+            }
+        }
+        else
+        {
+            // nonBlocking
+
+            if (!contiguous<Type>())
+            {
+                FatalErrorIn("globalIndex::gather(..)")
+                    << "nonBlocking not supported for non-contiguous data"
+                    << exit(FatalError);
+            }
+
+            label startOfRequests = Pstream::nRequests();
+
+            // Set up write
             OPstream::write
             (
-                Pstream::scheduled,
+                commsType,
                 procIDs[0],
                 reinterpret_cast<const char*>(fld.begin()),
                 fld.byteSize(),
                 tag,
                 comm
             );
-        }
-        else
-        {
-            OPstream toMaster
-            (
-                Pstream::scheduled,
-                procIDs[0],
-                0,
-                tag,
-                comm
-            );
-            toMaster << fld;
+
+            // Wait for all to finish
+            Pstream::waitRequests(startOfRequests);
         }
     }
 }
@@ -108,111 +174,104 @@ void Foam::globalIndex::gather
 template<class Type>
 void Foam::globalIndex::gather
 (
-    const labelUList& offsets,
+    const labelUList& off,
     const label comm,
     const labelList& procIDs,
     List<Type>& fld,
-    const int tag
+    const int tag,
+    const Pstream::commsTypes commsType
 )
 {
+    List<Type> allFld;
+
+    gather(off, comm, procIDs, fld, allFld, tag, commsType);
+
     if (Pstream::myProcNo(comm) == procIDs[0])
     {
-        List<Type> allFld(offsets.last());
-
-        // Assign my local data
-        SubList<Type>(allFld, fld.size(), 0).assign(fld);
-
-        for (label i = 1; i < procIDs.size(); i++)
-        {
-            SubList<Type> procSlot(allFld, offsets[i+1]-offsets[i], offsets[i]);
-
-            if (contiguous<Type>())
-            {
-                IPstream::read
-                (
-                    Pstream::scheduled,
-                    procIDs[i],
-                    reinterpret_cast<char*>(procSlot.begin()),
-                    procSlot.byteSize(),
-                    tag,
-                    comm
-                );
-            }
-            else
-            {
-                IPstream fromSlave
-                (
-                    Pstream::scheduled,
-                    procIDs[i],
-                    0,
-                    tag,
-                    comm
-                );
-                fromSlave >> procSlot;
-            }
-        }
-
         fld.transfer(allFld);
     }
-    else
-    {
-        if (contiguous<Type>())
-        {
-            OPstream::write
-            (
-                Pstream::scheduled,
-                procIDs[0],
-                reinterpret_cast<const char*>(fld.begin()),
-                fld.byteSize(),
-                tag,
-                comm
-            );
-        }
-        else
-        {
-            OPstream toMaster
-            (
-                Pstream::scheduled,
-                procIDs[0],
-                0,
-                tag,
-                comm
-            );
-            toMaster << fld;
-        }
-    }
 }
 
 
 template<class Type>
 void Foam::globalIndex::scatter
 (
-    const labelUList& offsets,
+    const labelUList& off,
     const label comm,
     const labelList& procIDs,
     const UList<Type>& allFld,
     UList<Type>& fld,
-    const int tag
+    const int tag,
+    const Pstream::commsTypes commsType
 )
 {
     if (Pstream::myProcNo(comm) == procIDs[0])
     {
-        fld.assign(SubList<Type>(allFld, offsets[1]-offsets[0]));
+        fld.assign(SubList<Type>(allFld, off[1]-off[0]));
 
-        for (label i = 1; i < procIDs.size(); i++)
+        if (commsType == Pstream::scheduled || commsType == Pstream::blocking)
         {
-            const SubList<Type> procSlot
-            (
-                allFld,
-                offsets[i+1]-offsets[i],
-                offsets[i]
-            );
-
-            if (contiguous<Type>())
+            for (label i = 1; i < procIDs.size(); i++)
             {
+                const SubList<Type> procSlot
+                (
+                    allFld,
+                    off[i+1]-off[i],
+                    off[i]
+                );
+
+                if (contiguous<Type>())
+                {
+                    OPstream::write
+                    (
+                        commsType,
+                        procIDs[i],
+                        reinterpret_cast<const char*>(procSlot.begin()),
+                        procSlot.byteSize(),
+                        tag,
+                        comm
+                    );
+                }
+                else
+                {
+                    OPstream toSlave
+                    (
+                        commsType,
+                        procIDs[i],
+                        0,
+                        tag,
+                        comm
+                    );
+                    toSlave << procSlot;
+                }
+            }
+        }
+        else
+        {
+            // nonBlocking
+
+            if (!contiguous<Type>())
+            {
+                FatalErrorIn("globalIndex::scatter(..)")
+                    << "nonBlocking not supported for non-contiguous data"
+                    << exit(FatalError);
+            }
+
+            label startOfRequests = Pstream::nRequests();
+
+            // Set up writes
+            for (label i = 1; i < procIDs.size(); i++)
+            {
+                const SubList<Type> procSlot
+                (
+                    allFld,
+                    off[i+1]-off[i],
+                    off[i]
+                );
+
                 OPstream::write
                 (
-                    Pstream::scheduled,
+                    commsType,
                     procIDs[i],
                     reinterpret_cast<const char*>(procSlot.begin()),
                     procSlot.byteSize(),
@@ -220,45 +279,66 @@ void Foam::globalIndex::scatter
                     comm
                 );
             }
-            else
-            {
-                OPstream toSlave
-                (
-                    Pstream::scheduled,
-                    procIDs[i],
-                    0,
-                    tag,
-                    comm
-                );
-                toSlave << procSlot;
-            }
+
+            // Wait for all to finish
+            Pstream::waitRequests(startOfRequests);
         }
     }
     else
     {
-        if (contiguous<Type>())
+        if (commsType == Pstream::scheduled || commsType == Pstream::blocking)
         {
+            if (contiguous<Type>())
+            {
+                IPstream::read
+                (
+                    commsType,
+                    procIDs[0],
+                    reinterpret_cast<char*>(fld.begin()),
+                    fld.byteSize(),
+                    tag,
+                    comm
+                );
+            }
+            else
+            {
+                IPstream fromMaster
+                (
+                    commsType,
+                    procIDs[0],
+                    0,
+                    tag,
+                    comm
+                );
+                fromMaster >> fld;
+            }
+        }
+        else
+        {
+            // nonBlocking
+
+            if (!contiguous<Type>())
+            {
+                FatalErrorIn("globalIndex::scatter(..)")
+                    << "nonBlocking not supported for non-contiguous data"
+                    << exit(FatalError);
+            }
+
+            label startOfRequests = Pstream::nRequests();
+
+            // Set up read
             IPstream::read
             (
-                Pstream::scheduled,
+                commsType,
                 procIDs[0],
                 reinterpret_cast<char*>(fld.begin()),
                 fld.byteSize(),
                 tag,
                 comm
             );
-        }
-        else
-        {
-            IPstream fromMaster
-            (
-                Pstream::scheduled,
-                procIDs[0],
-                0,
-                tag,
-                comm
-            );
-            fromMaster >> fld;
+
+            // Wait for all to finish
+            Pstream::waitRequests(startOfRequests);
         }
     }
 }

From 0a6b9a36d66e831ccd9e70b99d6d259bd6091bba Mon Sep 17 00:00:00 2001
From: mattijs <mattijs>
Date: Fri, 10 May 2013 16:17:24 +0100
Subject: [PATCH 3/3] ENH: OBJstream: add line with normals

---
 src/surfMesh/surfaceFormats/obj/OBJstream.C | 16 +++++++++++++++-
 src/surfMesh/surfaceFormats/obj/OBJstream.H | 12 ++++++++++--
 2 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/src/surfMesh/surfaceFormats/obj/OBJstream.C b/src/surfMesh/surfaceFormats/obj/OBJstream.C
index ce0eb2721b..46a6b7e0c0 100644
--- a/src/surfMesh/surfaceFormats/obj/OBJstream.C
+++ b/src/surfMesh/surfaceFormats/obj/OBJstream.C
@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2012 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2012-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -241,6 +241,20 @@ Foam::Ostream& Foam::OBJstream::write(const linePointRef& ln)
 }
 
 
+Foam::Ostream& Foam::OBJstream::write
+(
+    const linePointRef& ln,
+    const vector& n0,
+    const vector& n1
+)
+{
+    write(ln.start(), n0);
+    write(ln.end(), n1);
+    write("l ") << nVertices_-1 << ' ' << nVertices_ << nl;
+    return *this;
+}
+
+
 Foam::Ostream& Foam::OBJstream::write
 (
     const triPointRef& f,
diff --git a/src/surfMesh/surfaceFormats/obj/OBJstream.H b/src/surfMesh/surfaceFormats/obj/OBJstream.H
index ec7e93d24e..ee6ae58143 100644
--- a/src/surfMesh/surfaceFormats/obj/OBJstream.H
+++ b/src/surfMesh/surfaceFormats/obj/OBJstream.H
@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2012 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2012-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -92,7 +92,7 @@ public:
 
         // Access
 
-            //- Return the name of the stream
+            //- Return the number of vertices written
             label nVertices() const
             {
                 return nVertices_;
@@ -135,6 +135,14 @@ public:
             //- Write line
             Ostream& write(const linePointRef&);
 
+            //- Write line with points and vector normals ('vn')
+            Ostream& write
+            (
+                const linePointRef&,
+                const vector& n0,
+                const vector& n1
+            );
+
             //- Write triangle as points with lines or filled polygon
             Ostream& write(const triPointRef&, const bool lines = true);