From 7605c01260fb53ff7e98f8d53d74328febef46bf Mon Sep 17 00:00:00 2001
From: mattijs <mattijs>
Date: Thu, 22 Mar 2012 12:39:59 +0000
Subject: [PATCH] ENH: gAverage: (used in normFactor) : use single reduction

---
 .../db/IOstreams/Pstreams/PstreamReduceOps.H  |  21 +++
 .../fields/Fields/Field/FieldFunctions.C      |   7 +-
 src/Pstream/dummy/UPstream.C                  |   9 ++
 src/Pstream/mpi/UPstream.C                    | 138 +++++++++++++++++-
 4 files changed, 171 insertions(+), 4 deletions(-)

diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H
index b04e77a514..f6f69bba9e 100644
--- a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H
+++ b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H
@@ -96,6 +96,20 @@ T returnReduce
 }
 
 
+// Reduce with sum of both value and count (for averaging)
+template <class T>
+void sumReduce
+(
+    T& Value,
+    label& Count,
+    const int tag = Pstream::msgType()
+)
+{
+    reduce(Value, sumOp<T>(), tag);
+    reduce(Count, sumOp<label>(), tag);
+}
+
+
 // Non-blocking version of reduce. Sets request.
 template <class T, class BinaryOp>
 void reduce
@@ -125,6 +139,13 @@ void reduce
     const int tag = Pstream::msgType()
 );
 
+void sumReduce
+(
+    scalar& Value,
+    label& Count,
+    const int tag = Pstream::msgType()
+);
+
 void reduce
 (
     scalar& Value,
diff --git a/src/OpenFOAM/fields/Fields/Field/FieldFunctions.C b/src/OpenFOAM/fields/Fields/Field/FieldFunctions.C
index 193651b520..9397411ec4 100644
--- a/src/OpenFOAM/fields/Fields/Field/FieldFunctions.C
+++ b/src/OpenFOAM/fields/Fields/Field/FieldFunctions.C
@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2012 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -514,11 +514,12 @@ template<class Type>
 Type gAverage(const UList<Type>& f)
 {
     label n = f.size();
-    reduce(n, sumOp<label>());
+    Type s = sum(f);
+    sumReduce(s, n);
 
     if (n > 0)
     {
-        Type avrg = gSum(f)/n;
+        Type avrg = s/n;
 
         return avrg;
     }
diff --git a/src/Pstream/dummy/UPstream.C b/src/Pstream/dummy/UPstream.C
index 3c1bd813c6..d787291f1d 100644
--- a/src/Pstream/dummy/UPstream.C
+++ b/src/Pstream/dummy/UPstream.C
@@ -63,6 +63,15 @@ void Foam::reduce(vector2D&, const sumOp<vector2D>&, const int)
 {}
 
 
+void Foam::sumReduce
+(
+    scalar& Value,
+    label& Count,
+    const int tag
+)
+{}
+
+
 void Foam::reduce(scalar&, const sumOp<scalar>&, const int, label&)
 {}
 
diff --git a/src/Pstream/mpi/UPstream.C b/src/Pstream/mpi/UPstream.C
index 11d654b61f..e5b2808ca6 100644
--- a/src/Pstream/mpi/UPstream.C
+++ b/src/Pstream/mpi/UPstream.C
@@ -41,7 +41,38 @@ License
 #   define MPI_SCALAR MPI_DOUBLE
 #endif
 
-// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
+
+namespace Foam
+{
+    typedef struct
+    {
+        scalar value;
+        label count;
+    } CountAndValue;
+
+    void reduceSum
+    (
+        void *in,
+        void *inOut,
+        int *len,
+        MPI_Datatype *dptr
+    )
+    {
+        CountAndValue* inPtr =
+            reinterpret_cast<CountAndValue*>(in);
+        CountAndValue* inOutPtr =
+            reinterpret_cast<CountAndValue*>(inOut);
+
+        for (int i=0; i< *len; ++i)
+        {
+            inOutPtr->value += inPtr->value;
+            inOutPtr->count += inPtr->count;
+            inPtr++;
+            inOutPtr++;
+        }
+    }
+}
 
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
@@ -455,6 +486,111 @@ void Foam::reduce(vector2D& Value, const sumOp<vector2D>& bop, const int tag)
 }
 
 
+void Foam::sumReduce
+(
+    scalar& Value,
+    label& Count,
+    const int tag
+)
+{
+    static bool hasDataType_ = false;
+    static MPI_Datatype mesg_mpi_strct_;
+    static MPI_Op myOp_;
+
+    if (Pstream::debug)
+    {
+        Pout<< "Foam::sumReduce : value:" << Value
+            << " count:" << Count << endl;
+    }
+
+    if (!UPstream::parRun())
+    {
+        return;
+    }
+
+    if (UPstream::nProcs() <= UPstream::nProcsSimpleSum)
+    {
+        reduce(Value, sumOp<scalar>(), tag);
+        reduce(Count, sumOp<label>(), tag);
+    }
+    else
+    {
+        CountAndValue in,out;
+
+        if (!hasDataType_)
+        {
+            int lengths[2];
+            lengths[0] = 1;
+            lengths[1] = 1;
+            MPI_Datatype types[2];
+            types[0] = MPI_DOUBLE;
+            types[1] = MPI_INT;
+            MPI_Aint addresses[2];
+            MPI_Address(&in.value, &addresses[0]);
+            MPI_Address(&in.count, &addresses[1]);
+            MPI_Aint offsets[2];
+            offsets[0] = 0;
+            offsets[1] = addresses[1]-addresses[0];
+
+            if
+            (
+                MPI_Type_create_struct
+                (
+                    2,
+                    lengths,
+                    offsets,
+                    types,
+                    &mesg_mpi_strct_
+                )
+            )
+            {
+                FatalErrorIn("Foam::sumReduce()")
+                    << "MPI_Type_create_struct" << abort(FatalError);
+            }
+            if (MPI_Type_commit(&mesg_mpi_strct_))
+            {
+                FatalErrorIn("Foam::sumReduce()")
+                    << "MPI_Type_commit" << abort(FatalError);
+            }
+            if (MPI_Op_create(reduceSum, true, &myOp_))
+            {
+                FatalErrorIn("Foam::sumReduce()")
+                    << "MPI_Op_create" << abort(FatalError);
+            }
+
+            hasDataType_ = true;
+        }
+
+        in.value = Value;
+        in.count = Count;
+        if
+        (
+            MPI_Allreduce
+            (
+                &in,
+                &out,
+                1,
+                mesg_mpi_strct_,
+                myOp_,
+                MPI_COMM_WORLD
+            )
+        )
+        {
+            FatalErrorIn("Foam::sumReduce(..)")
+                << "Problem." << abort(FatalError);
+        }
+        Value = out.value;
+        Count = out.count;
+    }
+
+    if (Pstream::debug)
+    {
+        Pout<< "Foam::reduce : reduced value:" << Value
+            << " reduced count:" << Count << endl;
+    }
+}
+
+
 void Foam::reduce
 (
     scalar& Value,