From c983a581d49f598d7bdef86887b34cc1808b1666 Mon Sep 17 00:00:00 2001
From: mattijs <mattijs>
Date: Sat, 20 Apr 2013 20:59:24 +0100
Subject: [PATCH] ENH: GAMGProcAgglomeration: check for parallel running

---
 src/OpenFOAM/Make/files                       |   2 +
 .../GAMGAgglomeration/GAMGAgglomeration.C     |   5 +-
 .../eagerGAMGProcAgglomeration.C              | 168 ++++++++++++++++++
 .../eagerGAMGProcAgglomeration.H              | 113 ++++++++++++
 .../manualGAMGProcAgglomeration.C             | 151 ++++++++--------
 .../masterCoarsestGAMGProcAgglomeration.C     |  66 +++----
 6 files changed, 402 insertions(+), 103 deletions(-)
 create mode 100644 src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/eagerGAMGProcAgglomeration/eagerGAMGProcAgglomeration.C
 create mode 100644 src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/eagerGAMGProcAgglomeration/eagerGAMGProcAgglomeration.H

diff --git a/src/OpenFOAM/Make/files b/src/OpenFOAM/Make/files
index 321855de69..554e890db9 100644
--- a/src/OpenFOAM/Make/files
+++ b/src/OpenFOAM/Make/files
@@ -328,6 +328,8 @@ masterCoarsestGAMGProcAgglomeration = $(GAMGProcAgglomerations)/masterCoarsestGA
 $(masterCoarsestGAMGProcAgglomeration)/masterCoarsestGAMGProcAgglomeration.C
 manualGAMGProcAgglomeration = $(GAMGProcAgglomerations)/manualGAMGProcAgglomeration
 $(manualGAMGProcAgglomeration)/manualGAMGProcAgglomeration.C
+eagerGAMGProcAgglomeration = $(GAMGProcAgglomerations)/eagerGAMGProcAgglomeration
+$(eagerGAMGProcAgglomeration)/eagerGAMGProcAgglomeration.C
 /*
 cellFaceRatioGAMGProcAgglomeration = $(GAMGProcAgglomerations)/cellFaceRatioGAMGProcAgglomeration
 $(cellFaceRatioGAMGProcAgglomeration)/cellFaceRatioGAMGProcAgglomeration.C
diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomeration.C b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomeration.C
index 324ace4c91..a572f5ff43 100644
--- a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomeration.C
+++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomeration.C
@@ -121,7 +121,10 @@ Foam::GAMGAgglomeration::GAMGAgglomeration
     ),
     procAgglomeratorPtr_
     (
-        controlDict.found("processorAgglomerator")
+        (
+            (UPstream::nProcs(mesh.comm()) > 1)
+         && controlDict.found("processorAgglomerator")
+        )
       ? GAMGProcAgglomeration::New
         (
             controlDict.lookup("processorAgglomerator"),
diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/eagerGAMGProcAgglomeration/eagerGAMGProcAgglomeration.C b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/eagerGAMGProcAgglomeration/eagerGAMGProcAgglomeration.C
new file mode 100644
index 0000000000..c1f166d1ec
--- /dev/null
+++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/eagerGAMGProcAgglomeration/eagerGAMGProcAgglomeration.C
@@ -0,0 +1,168 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | Copyright (C) 2013 OpenFOAM Foundation
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "eagerGAMGProcAgglomeration.H"
+#include "addToRunTimeSelectionTable.H"
+#include "GAMGAgglomeration.H"
+
+// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
+
+namespace Foam
+{
+    defineTypeNameAndDebug(eagerGAMGProcAgglomeration, 0);
+
+    addToRunTimeSelectionTable
+    (
+        GAMGProcAgglomeration,
+        eagerGAMGProcAgglomeration,
+        GAMGAgglomeration
+    );
+}
+
+
+// * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
+
+
+// * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
+
+Foam::eagerGAMGProcAgglomeration::eagerGAMGProcAgglomeration
+(
+    GAMGAgglomeration& agglom,
+    const dictionary& controlDict
+)
+:
+    GAMGProcAgglomeration(agglom, controlDict),
+    mergeLevels_(controlDict.lookupOrDefault<label>("mergeLevels", 1))
+{}
+
+
+// * * * * * * * * * * * * * * * * Destructor  * * * * * * * * * * * * * * * //
+
+Foam::eagerGAMGProcAgglomeration::
+~eagerGAMGProcAgglomeration()
+{
+    forAllReverse(comms_, i)
+    {
+        if (comms_[i] != -1)
+        {
+            UPstream::freeCommunicator(comms_[i]);
+        }
+    }
+}
+
+
+// * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
+
+bool Foam::eagerGAMGProcAgglomeration::agglomerate()
+{
+    if (debug)
+    {
+        Pout<< nl << "Starting mesh overview" << endl;
+        printStats(Pout, agglom_);
+    }
+
+    if (agglom_.size() >= 1)
+    {
+        // Agglomerate one but last level (since also agglomerating
+        // restrictAddressing)
+        for
+        (
+            label fineLevelIndex = 2;
+            fineLevelIndex < agglom_.size();
+            fineLevelIndex++
+        )
+        {
+            if (agglom_.hasMeshLevel(fineLevelIndex))
+            {
+                // Get the fine mesh
+                const lduMesh& levelMesh = agglom_.meshLevel(fineLevelIndex);
+                label levelComm = levelMesh.comm();
+                label nProcs = UPstream::nProcs(levelComm);
+
+                if (nProcs > 1)
+                {
+                    // Processor restriction map: per processor the coarse
+                    // processor
+                    labelList procAgglomMap(nProcs);
+
+                    forAll(procAgglomMap, procI)
+                    {
+                        procAgglomMap[procI] = procI/(1<<mergeLevels_);
+                    }
+
+                    // Master processor
+                    labelList masterProcs;
+                    // Local processors that agglomerate. agglomProcIDs[0]
+                    // is in masterProc.
+                    List<int> agglomProcIDs;
+                    GAMGAgglomeration::calculateRegionMaster
+                    (
+                        levelComm,
+                        procAgglomMap,
+                        masterProcs,
+                        agglomProcIDs
+                    );
+
+                    // Allocate a communicator for the processor-agglomerated
+                    // matrix
+                    comms_.append
+                    (
+                        UPstream::allocateCommunicator
+                        (
+                            levelComm,
+                            masterProcs
+                        )
+                    );
+
+                    // Use procesor agglomeration maps to do the actual
+                    // collecting.
+                    if (Pstream::myProcNo(levelComm) != -1)
+                    {
+                        GAMGProcAgglomeration::agglomerate
+                        (
+                            fineLevelIndex,
+                            procAgglomMap,
+                            masterProcs,
+                            agglomProcIDs,
+                            comms_.last()
+                        );
+                    }
+                }
+            }
+        }
+    }
+
+    // Print a bit
+    if (debug)
+    {
+        Pout<< nl << "Agglomerated mesh overview" << endl;
+        printStats(Pout, agglom_);
+    }
+
+    return true;
+}
+
+
+// ************************************************************************* //
diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/eagerGAMGProcAgglomeration/eagerGAMGProcAgglomeration.H b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/eagerGAMGProcAgglomeration/eagerGAMGProcAgglomeration.H
new file mode 100644
index 0000000000..99bc37941c
--- /dev/null
+++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/eagerGAMGProcAgglomeration/eagerGAMGProcAgglomeration.H
@@ -0,0 +1,113 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | Copyright (C) 2013 OpenFOAM Foundation
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+Class
+    Foam::eagerGAMGProcAgglomeration
+
+Description
+    'Eager' processor agglomeration of GAMGAgglomerations: at every
+    level agglomerates 'mergeLevels' number of processors onto the
+    minimum processor number.
+
+SourceFiles
+    eagerGAMGProcAgglomeration.C
+
+\*---------------------------------------------------------------------------*/
+
+#ifndef eagerGAMGProcAgglomeration_H
+#define eagerGAMGProcAgglomeration_H
+
+#include "GAMGProcAgglomeration.H"
+#include "DynamicList.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+class GAMGAgglomeration;
+
+/*---------------------------------------------------------------------------*\
+              Class eagerGAMGProcAgglomeration Declaration
+\*---------------------------------------------------------------------------*/
+
+class eagerGAMGProcAgglomeration
+:
+    public GAMGProcAgglomeration
+{
+    // Private data
+
+        //- Agglpmeration level
+        const label mergeLevels_;
+
+        DynamicList<label> comms_;
+
+    // Private Member Functions
+
+        //- Disallow default bitwise copy construct
+        eagerGAMGProcAgglomeration
+        (
+            const eagerGAMGProcAgglomeration&
+        );
+
+        //- Disallow default bitwise assignment
+        void operator=(const eagerGAMGProcAgglomeration&);
+
+
+public:
+
+    //- Runtime type information
+    TypeName("eager");
+
+
+    // Constructors
+
+        //- Construct given agglomerator and controls
+        eagerGAMGProcAgglomeration
+        (
+            GAMGAgglomeration& agglom,
+            const dictionary& controlDict
+        );
+
+
+    //- Destructor
+    virtual ~eagerGAMGProcAgglomeration();
+
+
+    // Member Functions
+
+       //- Modify agglomeration. Return true if modified
+        virtual bool agglomerate();
+
+};
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace Foam
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+#endif
+
+// ************************************************************************* //
diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/manualGAMGProcAgglomeration/manualGAMGProcAgglomeration.C b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/manualGAMGProcAgglomeration/manualGAMGProcAgglomeration.C
index b2f968aaf6..1fea00057c 100644
--- a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/manualGAMGProcAgglomeration/manualGAMGProcAgglomeration.C
+++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/manualGAMGProcAgglomeration/manualGAMGProcAgglomeration.C
@@ -102,95 +102,104 @@ bool Foam::manualGAMGProcAgglomeration::agglomerate()
             {
                 // Get the fine mesh
                 const lduMesh& levelMesh = agglom_.meshLevel(fineLevelIndex);
+                label nProcs = UPstream::nProcs(levelMesh.comm());
 
-                // My processor id
-                const label myProcID = Pstream::myProcNo(levelMesh.comm());
-
-                const List<clusterAndMaster>& clusters =
-                    procAgglomMaps_[i].second();
-
-                // Coarse to fine master processor
-                labelList coarseToMaster(clusters.size());
-
-                // Fine to coarse map
-                labelList procAgglomMap(Pstream::nProcs(levelMesh.comm()), -1);
-
-                // Cluster for my processor (with master index first)
-                labelList agglomProcIDs;
-
-
-
-                forAll(clusters, coarseI)
+                if (nProcs > 1)
                 {
-                    const labelList& cluster = clusters[coarseI].first();
-                    coarseToMaster[coarseI] = clusters[coarseI].second();
+                    // My processor id
+                    const label myProcID = Pstream::myProcNo(levelMesh.comm());
 
-                    forAll(cluster, i)
+                    const List<clusterAndMaster>& clusters =
+                        procAgglomMaps_[i].second();
+
+                    // Coarse to fine master processor
+                    labelList coarseToMaster(clusters.size());
+
+                    // Fine to coarse map
+                    labelList procAgglomMap(nProcs, -1);
+
+                    // Cluster for my processor (with master index first)
+                    labelList agglomProcIDs;
+
+
+
+                    forAll(clusters, coarseI)
                     {
-                        procAgglomMap[cluster[i]] = coarseI;
+                        const labelList& cluster = clusters[coarseI].first();
+                        coarseToMaster[coarseI] = clusters[coarseI].second();
+
+                        forAll(cluster, i)
+                        {
+                            procAgglomMap[cluster[i]] = coarseI;
+                        }
+
+                        label masterIndex = findIndex
+                        (
+                            cluster,
+                            coarseToMaster[coarseI]
+                        );
+
+                        if (masterIndex == -1)
+                        {
+                            FatalErrorIn
+                            (
+                                "manualGAMGProcAgglomeration::agglomerate()"
+                            )   << "At level " << fineLevelIndex
+                                << " the master processor "
+                                << coarseToMaster[coarseI]
+                                << " is not in the cluster "
+                                << cluster
+                                << exit(FatalError);
+                        }
+
+                        if (findIndex(cluster, myProcID) != -1)
+                        {
+                            // This is my cluster. Make sure master index is
+                            // first
+                            agglomProcIDs = cluster;
+                            Swap(agglomProcIDs[0], agglomProcIDs[masterIndex]);
+                        }
                     }
 
-                    label masterIndex = findIndex
-                    (
-                        cluster,
-                        coarseToMaster[coarseI]
-                    );
 
-                    if (masterIndex == -1)
+                    // Check that we've done all processors
+                    if (findIndex(procAgglomMap, -1) != -1)
                     {
                         FatalErrorIn
                         (
                             "manualGAMGProcAgglomeration::agglomerate()"
                         )   << "At level " << fineLevelIndex
-                            << " the master processor "
-                            << coarseToMaster[coarseI]
-                            << " is not in the cluster "
-                            << cluster
+                            << " processor "
+                            << findIndex(procAgglomMap, -1)
+                            << " is not in any cluster"
                             << exit(FatalError);
                     }
 
-                    if (findIndex(cluster, myProcID) != -1)
-                    {
-                        // This is my cluster. Make sure master index is first
-                        agglomProcIDs = cluster;
-                        Swap(agglomProcIDs[0], agglomProcIDs[masterIndex]);
-                    }
-                }
 
-
-                // Check that we've done all processors
-                if (findIndex(procAgglomMap, -1) != -1)
-                {
-                    FatalErrorIn("manualGAMGProcAgglomeration::agglomerate()")
-                        << "At level " << fineLevelIndex
-                        << " processor "
-                        << findIndex(procAgglomMap, -1)
-                        << " is not in any cluster"
-                        << exit(FatalError);
-                }
-
-
-                // Allocate a communicator for the processor-agglomerated matrix
-                comms_.append
-                (
-                    UPstream::allocateCommunicator
+                    // Allocate a communicator for the processor-agglomerated
+                    // matrix
+                    comms_.append
                     (
-                        levelMesh.comm(),
-                        coarseToMaster
-                    )
-                );
-
-                // Use procesor agglomeration maps to do the actual collecting.
-                if (Pstream::myProcNo(levelMesh.comm()) != -1)
-                {
-                    GAMGProcAgglomeration::agglomerate
-                    (
-                        fineLevelIndex,
-                        procAgglomMap,
-                        coarseToMaster,
-                        agglomProcIDs,
-                        comms_.last()
+                        UPstream::allocateCommunicator
+                        (
+                            levelMesh.comm(),
+                            coarseToMaster
+                        )
                     );
+
+                    // Use procesor agglomeration maps to do the actual
+                    // collecting
+                    if (Pstream::myProcNo(levelMesh.comm()) != -1)
+                    {
+                        GAMGProcAgglomeration::agglomerate
+                        (
+                            fineLevelIndex,
+                            procAgglomMap,
+                            coarseToMaster,
+                            agglomProcIDs,
+                            comms_.last()
+                        );
+                    }
                 }
             }
         }
diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/masterCoarsestGAMGProcAgglomeration/masterCoarsestGAMGProcAgglomeration.C b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/masterCoarsestGAMGProcAgglomeration/masterCoarsestGAMGProcAgglomeration.C
index 8f3552c9ff..480795fb55 100644
--- a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/masterCoarsestGAMGProcAgglomeration/masterCoarsestGAMGProcAgglomeration.C
+++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/masterCoarsestGAMGProcAgglomeration/masterCoarsestGAMGProcAgglomeration.C
@@ -93,44 +93,48 @@ bool Foam::masterCoarsestGAMGProcAgglomeration::agglomerate()
             // Get the fine mesh
             const lduMesh& levelMesh = agglom_.meshLevel(fineLevelIndex);
             label levelComm = levelMesh.comm();
+            label nProcs = UPstream::nProcs(levelComm);
 
-            // Processor restriction map: per processor the coarse processor
-            labelList procAgglomMap(UPstream::nProcs(levelComm), 0);
+            if (nProcs > 1)
+            {
+                // Processor restriction map: per processor the coarse processor
+                labelList procAgglomMap(nProcs, 0);
 
-            // Master processor
-            labelList masterProcs;
-            // Local processors that agglomerate. agglomProcIDs[0] is in
-            // masterProc.
-            List<int> agglomProcIDs;
-            GAMGAgglomeration::calculateRegionMaster
-            (
-                levelComm,
-                procAgglomMap,
-                masterProcs,
-                agglomProcIDs
-            );
-
-            // Allocate a communicator for the processor-agglomerated matrix
-            comms_.append
-            (
-                UPstream::allocateCommunicator
+                // Master processor
+                labelList masterProcs;
+                // Local processors that agglomerate. agglomProcIDs[0] is in
+                // masterProc.
+                List<int> agglomProcIDs;
+                GAMGAgglomeration::calculateRegionMaster
                 (
                     levelComm,
-                    masterProcs
-                )
-            );
-
-            // Use procesor agglomeration maps to do the actual collecting.
-            if (Pstream::myProcNo(levelComm) != -1)
-            {
-                GAMGProcAgglomeration::agglomerate
-                (
-                    fineLevelIndex,
                     procAgglomMap,
                     masterProcs,
-                    agglomProcIDs,
-                    comms_.last()
+                    agglomProcIDs
                 );
+
+                // Allocate a communicator for the processor-agglomerated matrix
+                comms_.append
+                (
+                    UPstream::allocateCommunicator
+                    (
+                        levelComm,
+                        masterProcs
+                    )
+                );
+
+                // Use procesor agglomeration maps to do the actual collecting.
+                if (Pstream::myProcNo(levelComm) != -1)
+                {
+                    GAMGProcAgglomeration::agglomerate
+                    (
+                        fineLevelIndex,
+                        procAgglomMap,
+                        masterProcs,
+                        agglomProcIDs,
+                        comms_.last()
+                    );
+                }
             }
         }
     }