From 62244c6caf54bd99e73cde360f1e295cfc40d371 Mon Sep 17 00:00:00 2001
From: Mattijs Janssens <ext-mjanssens@esi-group.com>
Date: Thu, 20 Oct 2022 09:14:22 +0000
Subject: [PATCH] Feature master coarsest multi masters

---
 .../masterCoarsestGAMGProcAgglomeration.C     | 93 ++++++++++++++++++-
 .../masterCoarsestGAMGProcAgglomeration.H     | 26 ++++++
 2 files changed, 114 insertions(+), 5 deletions(-)

diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/masterCoarsestGAMGProcAgglomeration/masterCoarsestGAMGProcAgglomeration.C b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/masterCoarsestGAMGProcAgglomeration/masterCoarsestGAMGProcAgglomeration.C
index 5e9ad314ad..20af4576cd 100644
--- a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/masterCoarsestGAMGProcAgglomeration/masterCoarsestGAMGProcAgglomeration.C
+++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/masterCoarsestGAMGProcAgglomeration/masterCoarsestGAMGProcAgglomeration.C
@@ -6,6 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2013-2014 OpenFOAM Foundation
+    Copyright (C) 2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -52,8 +53,48 @@ Foam::masterCoarsestGAMGProcAgglomeration::masterCoarsestGAMGProcAgglomeration
     const dictionary& controlDict
 )
 :
-    GAMGProcAgglomeration(agglom, controlDict)
-{}
+    GAMGProcAgglomeration(agglom, controlDict),
+    nProcessorsPerMaster_
+    (
+        controlDict.getOrDefault<label>
+        (
+            "nProcessorsPerMaster",
+            0,
+            keyType::LITERAL
+        )
+    )
+{
+    const auto* ePtr = controlDict.findEntry("nMasters", keyType::LITERAL);
+    if (ePtr)
+    {
+        if (nProcessorsPerMaster_ > 0)
+        {
+            FatalIOErrorInFunction(controlDict)
+                << "Cannot specify both \"nMasters\" and"
+                << " \"nProcessorsPerMaster\"" << exit(FatalIOError);
+        }
+
+        const label nMasters(readLabel(ePtr->stream()));
+
+        if (nMasters <= 0)
+        {
+            FatalIOErrorInFunction(controlDict)
+                << "Illegal value \"nMasters\" "
+                << nMasters << exit(FatalIOError);
+        }
+
+        nProcessorsPerMaster_ =
+            (Pstream::nProcs(agglom.mesh().comm())+nMasters-1)
+          / nMasters;
+    }
+
+    if (nProcessorsPerMaster_ < 0)
+    {
+        FatalIOErrorInFunction(controlDict)
+            << "Illegal value \"nProcessorsPerMaster\" "
+            << nProcessorsPerMaster_ << exit(FatalIOError);
+    }
+}
 
 
 // * * * * * * * * * * * * * * * * Destructor  * * * * * * * * * * * * * * * //
@@ -75,7 +116,7 @@ Foam::masterCoarsestGAMGProcAgglomeration::
 
 bool Foam::masterCoarsestGAMGProcAgglomeration::agglomerate()
 {
-    if (debug)
+    if (debug & 2)
     {
         Pout<< nl << "Starting mesh overview" << endl;
         printStats(Pout, agglom_);
@@ -97,7 +138,23 @@ bool Foam::masterCoarsestGAMGProcAgglomeration::agglomerate()
             if (nProcs > 1)
             {
                 // Processor restriction map: per processor the coarse processor
-                labelList procAgglomMap(nProcs, Zero);
+                labelList procAgglomMap(nProcs);
+
+                if (nProcessorsPerMaster_ > 0)
+                {
+                    forAll(procAgglomMap, fineProci)
+                    {
+                        procAgglomMap[fineProci] =
+                        (
+                            fineProci
+                          / nProcessorsPerMaster_
+                        );
+                    }
+                }
+                else
+                {
+                    procAgglomMap = Zero;
+                }
 
                 // Master processor
                 labelList masterProcs;
@@ -112,6 +169,32 @@ bool Foam::masterCoarsestGAMGProcAgglomeration::agglomerate()
                     agglomProcIDs
                 );
 
+                if (debug)
+                {
+                    if (masterProcs.size())
+                    {
+                        labelListList masterToProcs
+                        (
+                            invertOneToMany
+                            (
+                                masterProcs.size(),
+                                procAgglomMap
+                            )
+                        );
+                        Info<< typeName << " : agglomerating" << nl
+                            << "\tmaster\tnProcs\tprocIDs" << endl;
+                        for (const auto& p : masterToProcs)
+                        {
+                            Info<< '\t' << p[0]
+                                << '\t' << p.size()
+                                << '\t'
+                                << flatOutput(SubList<label>(p, p.size()-1, 1))
+                                << endl;
+                        }
+                    }
+                }
+
+
                 // Allocate a communicator for the processor-agglomerated matrix
                 comms_.append
                 (
@@ -139,7 +222,7 @@ bool Foam::masterCoarsestGAMGProcAgglomeration::agglomerate()
     }
 
     // Print a bit
-    if (debug)
+    if (debug & 2)
     {
         Pout<< nl << "Agglomerated mesh overview" << endl;
         printStats(Pout, agglom_);
diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/masterCoarsestGAMGProcAgglomeration/masterCoarsestGAMGProcAgglomeration.H b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/masterCoarsestGAMGProcAgglomeration/masterCoarsestGAMGProcAgglomeration.H
index 38951ac91e..8daaafa685 100644
--- a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/masterCoarsestGAMGProcAgglomeration/masterCoarsestGAMGProcAgglomeration.H
+++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/masterCoarsestGAMGProcAgglomeration/masterCoarsestGAMGProcAgglomeration.H
@@ -6,6 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2013 OpenFOAM Foundation
+    Copyright (C) 2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -29,6 +30,28 @@ Class
 Description
     Processor agglomeration of GAMGAgglomerations.
 
+    - by default agglomerates onto the master processor
+    - optionally have multiple masters through the nProcessorsPerMaster
+      parameter. e.g.
+
+        p
+        {
+            solver                  GAMG;
+            tolerance               1e-06;
+            relTol                  0.1;
+            smoother                GaussSeidel;
+
+            nCellsInCoarsestLevel   10;
+
+            processorAgglomerator   masterCoarsest;
+            // Groups of 32 cores get combined so each 'master' gets
+            // (roughly) 32*nCellsInCoarsestLevel cells.
+            nProcessorsPerMaster    32;
+
+            // Alternative : specify number of (equi-distributed) masters
+            nMasters                2;
+        }
+
 SourceFiles
     masterCoarsestGAMGProcAgglomeration.C
 
@@ -57,8 +80,11 @@ class masterCoarsestGAMGProcAgglomeration
 {
     // Private data
 
+        label nProcessorsPerMaster_;
+
         DynamicList<label> comms_;
 
+
     // Private Member Functions
 
         //- No copy construct