ENH: GAMGProcAgglomeration: check for parallel running

src/OpenFOAM/Make/files

@@ -328,6 +328,8 @@ masterCoarsestGAMGProcAgglomeration = $(GAMGProcAgglomerations)/masterCoarsestGA
 $(masterCoarsestGAMGProcAgglomeration)/masterCoarsestGAMGProcAgglomeration.C
 manualGAMGProcAgglomeration = $(GAMGProcAgglomerations)/manualGAMGProcAgglomeration
 $(manualGAMGProcAgglomeration)/manualGAMGProcAgglomeration.C
+eagerGAMGProcAgglomeration = $(GAMGProcAgglomerations)/eagerGAMGProcAgglomeration
+$(eagerGAMGProcAgglomeration)/eagerGAMGProcAgglomeration.C
 /*
 cellFaceRatioGAMGProcAgglomeration = $(GAMGProcAgglomerations)/cellFaceRatioGAMGProcAgglomeration
 $(cellFaceRatioGAMGProcAgglomeration)/cellFaceRatioGAMGProcAgglomeration.C

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomeration.C

@@ -121,7 +121,10 @@ Foam::GAMGAgglomeration::GAMGAgglomeration
     ),
     procAgglomeratorPtr_
     (
-        controlDict.found("processorAgglomerator")
+        (
+            (UPstream::nProcs(mesh.comm()) > 1)
+         && controlDict.found("processorAgglomerator")
+        )
       ? GAMGProcAgglomeration::New
         (
             controlDict.lookup("processorAgglomerator"),

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/eagerGAMGProcAgglomeration/eagerGAMGProcAgglomeration.C

@@ -0,0 +1,168 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | Copyright (C) 2013 OpenFOAM Foundation
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "eagerGAMGProcAgglomeration.H"
+#include "addToRunTimeSelectionTable.H"
+#include "GAMGAgglomeration.H"
+
+// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
+
+namespace Foam
+{
+    defineTypeNameAndDebug(eagerGAMGProcAgglomeration, 0);
+
+    addToRunTimeSelectionTable
+    (
+        GAMGProcAgglomeration,
+        eagerGAMGProcAgglomeration,
+        GAMGAgglomeration
+    );
+}
+
+
+// * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
+
+
+// * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
+
+Foam::eagerGAMGProcAgglomeration::eagerGAMGProcAgglomeration
+(
+    GAMGAgglomeration& agglom,
+    const dictionary& controlDict
+)
+:
+    GAMGProcAgglomeration(agglom, controlDict),
+    mergeLevels_(controlDict.lookupOrDefault<label>("mergeLevels", 1))
+{}
+
+
+// * * * * * * * * * * * * * * * * Destructor  * * * * * * * * * * * * * * * //
+
+Foam::eagerGAMGProcAgglomeration::
+~eagerGAMGProcAgglomeration()
+{
+    forAllReverse(comms_, i)
+    {
+        if (comms_[i] != -1)
+        {
+            UPstream::freeCommunicator(comms_[i]);
+        }
+    }
+}
+
+
+// * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
+
+bool Foam::eagerGAMGProcAgglomeration::agglomerate()
+{
+    if (debug)
+    {
+        Pout<< nl << "Starting mesh overview" << endl;
+        printStats(Pout, agglom_);
+    }
+
+    if (agglom_.size() >= 1)
+    {
+        // Agglomerate one but last level (since also agglomerating
+        // restrictAddressing)
+        for
+        (
+            label fineLevelIndex = 2;
+            fineLevelIndex < agglom_.size();
+            fineLevelIndex++
+        )
+        {
+            if (agglom_.hasMeshLevel(fineLevelIndex))
+            {
+                // Get the fine mesh
+                const lduMesh& levelMesh = agglom_.meshLevel(fineLevelIndex);
+                label levelComm = levelMesh.comm();
+                label nProcs = UPstream::nProcs(levelComm);
+
+                if (nProcs > 1)
+                {
+                    // Processor restriction map: per processor the coarse
+                    // processor
+                    labelList procAgglomMap(nProcs);
+
+                    forAll(procAgglomMap, procI)
+                    {
+                        procAgglomMap[procI] = procI/(1<<mergeLevels_);
+                    }
+
+                    // Master processor
+                    labelList masterProcs;
+                    // Local processors that agglomerate. agglomProcIDs[0]
+                    // is in masterProc.
+                    List<int> agglomProcIDs;
+                    GAMGAgglomeration::calculateRegionMaster
+                    (
+                        levelComm,
+                        procAgglomMap,
+                        masterProcs,
+                        agglomProcIDs
+                    );
+
+                    // Allocate a communicator for the processor-agglomerated
+                    // matrix
+                    comms_.append
+                    (
+                        UPstream::allocateCommunicator
+                        (
+                            levelComm,
+                            masterProcs
+                        )
+                    );
+
+                    // Use procesor agglomeration maps to do the actual
+                    // collecting.
+                    if (Pstream::myProcNo(levelComm) != -1)
+                    {
+                        GAMGProcAgglomeration::agglomerate
+                        (
+                            fineLevelIndex,
+                            procAgglomMap,
+                            masterProcs,
+                            agglomProcIDs,
+                            comms_.last()
+                        );
+                    }
+                }
+            }
+        }
+    }
+
+    // Print a bit
+    if (debug)
+    {
+        Pout<< nl << "Agglomerated mesh overview" << endl;
+        printStats(Pout, agglom_);
+    }
+
+    return true;
+}
+
+
+// ************************************************************************* //

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/eagerGAMGProcAgglomeration/eagerGAMGProcAgglomeration.H

@@ -0,0 +1,113 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | Copyright (C) 2013 OpenFOAM Foundation
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+Class
+    Foam::eagerGAMGProcAgglomeration
+
+Description
+    'Eager' processor agglomeration of GAMGAgglomerations: at every
+    level agglomerates 'mergeLevels' number of processors onto the
+    minimum processor number.
+
+SourceFiles
+    eagerGAMGProcAgglomeration.C
+
+\*---------------------------------------------------------------------------*/
+
+#ifndef eagerGAMGProcAgglomeration_H
+#define eagerGAMGProcAgglomeration_H
+
+#include "GAMGProcAgglomeration.H"
+#include "DynamicList.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+class GAMGAgglomeration;
+
+/*---------------------------------------------------------------------------*\
+              Class eagerGAMGProcAgglomeration Declaration
+\*---------------------------------------------------------------------------*/
+
+class eagerGAMGProcAgglomeration
+:
+    public GAMGProcAgglomeration
+{
+    // Private data
+
+        //- Agglpmeration level
+        const label mergeLevels_;
+
+        DynamicList<label> comms_;
+
+    // Private Member Functions
+
+        //- Disallow default bitwise copy construct
+        eagerGAMGProcAgglomeration
+        (
+            const eagerGAMGProcAgglomeration&
+        );
+
+        //- Disallow default bitwise assignment
+        void operator=(const eagerGAMGProcAgglomeration&);
+
+
+public:
+
+    //- Runtime type information
+    TypeName("eager");
+
+
+    // Constructors
+
+        //- Construct given agglomerator and controls
+        eagerGAMGProcAgglomeration
+        (
+            GAMGAgglomeration& agglom,
+            const dictionary& controlDict
+        );
+
+
+    //- Destructor
+    virtual ~eagerGAMGProcAgglomeration();
+
+
+    // Member Functions
+
+       //- Modify agglomeration. Return true if modified
+        virtual bool agglomerate();
+
+};
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace Foam
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+#endif
+
+// ************************************************************************* //

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/manualGAMGProcAgglomeration/manualGAMGProcAgglomeration.C

@@ -102,7 +102,10 @@ bool Foam::manualGAMGProcAgglomeration::agglomerate()
             {
                 // Get the fine mesh
                 const lduMesh& levelMesh = agglom_.meshLevel(fineLevelIndex);
+                label nProcs = UPstream::nProcs(levelMesh.comm());
 
+                if (nProcs > 1)
+                {
                     // My processor id
                     const label myProcID = Pstream::myProcNo(levelMesh.comm());
 
@@ -113,7 +116,7 @@ bool Foam::manualGAMGProcAgglomeration::agglomerate()
                     labelList coarseToMaster(clusters.size());
 
                     // Fine to coarse map
-                labelList procAgglomMap(Pstream::nProcs(levelMesh.comm()), -1);
+                    labelList procAgglomMap(nProcs, -1);
 
                     // Cluster for my processor (with master index first)
                     labelList agglomProcIDs;
@@ -151,7 +154,8 @@ bool Foam::manualGAMGProcAgglomeration::agglomerate()
 
                         if (findIndex(cluster, myProcID) != -1)
                         {
-                        // This is my cluster. Make sure master index is first
+                            // This is my cluster. Make sure master index is
+                            // first
                             agglomProcIDs = cluster;
                             Swap(agglomProcIDs[0], agglomProcIDs[masterIndex]);
                         }
@@ -161,8 +165,10 @@ bool Foam::manualGAMGProcAgglomeration::agglomerate()
                     // Check that we've done all processors
                     if (findIndex(procAgglomMap, -1) != -1)
                     {
-                    FatalErrorIn("manualGAMGProcAgglomeration::agglomerate()")
-                        << "At level " << fineLevelIndex
+                        FatalErrorIn
+                        (
+                            "manualGAMGProcAgglomeration::agglomerate()"
+                        )   << "At level " << fineLevelIndex
                             << " processor "
                             << findIndex(procAgglomMap, -1)
                             << " is not in any cluster"
@@ -170,7 +176,8 @@ bool Foam::manualGAMGProcAgglomeration::agglomerate()
                     }
 
 
-                // Allocate a communicator for the processor-agglomerated matrix
+                    // Allocate a communicator for the processor-agglomerated
+                    // matrix
                     comms_.append
                     (
                         UPstream::allocateCommunicator
@@ -180,7 +187,8 @@ bool Foam::manualGAMGProcAgglomeration::agglomerate()
                         )
                     );
 
-                // Use procesor agglomeration maps to do the actual collecting.
+                    // Use procesor agglomeration maps to do the actual
+                    // collecting
                     if (Pstream::myProcNo(levelMesh.comm()) != -1)
                     {
                         GAMGProcAgglomeration::agglomerate
@@ -194,6 +202,7 @@ bool Foam::manualGAMGProcAgglomeration::agglomerate()
                     }
                 }
             }
+        }
 
         // Print a bit
         if (debug)

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/masterCoarsestGAMGProcAgglomeration/masterCoarsestGAMGProcAgglomeration.C

@@ -93,9 +93,12 @@ bool Foam::masterCoarsestGAMGProcAgglomeration::agglomerate()
             // Get the fine mesh
             const lduMesh& levelMesh = agglom_.meshLevel(fineLevelIndex);
             label levelComm = levelMesh.comm();
+            label nProcs = UPstream::nProcs(levelComm);
 
+            if (nProcs > 1)
+            {
                 // Processor restriction map: per processor the coarse processor
-            labelList procAgglomMap(UPstream::nProcs(levelComm), 0);
+                labelList procAgglomMap(nProcs, 0);
 
                 // Master processor
                 labelList masterProcs;
@@ -134,6 +137,7 @@ bool Foam::masterCoarsestGAMGProcAgglomeration::agglomerate()
                 }
             }
         }
+    }
 
     // Print a bit
     if (debug)