From 73e9b2804f03dfded194c0f1e4e1546a51ecc090 Mon Sep 17 00:00:00 2001
From: Henry Weller <http://openfoam.org>
Date: Fri, 2 Feb 2018 19:20:58 +0000
Subject: [PATCH] PstreamGlobals::MPI_COMM_FOAM: New OpenFOAM specific MPI
 communicator

Splitting MPI_COMM_FOAM from MPI_COMM_WORLD allows OpenFOAM to be linked with
other libraries communicating via MPI.

Resolves feature request https://bugs.openfoam.org/view.php?id=2815
---
 src/Pstream/mpi/PstreamGlobals.C              |  4 ++-
 src/Pstream/mpi/PstreamGlobals.H              | 21 ++++++------
 src/Pstream/mpi/UOPwrite.C                    |  8 ++---
 src/Pstream/mpi/UPstream.C                    | 32 +++++++++++++++----
 .../decompose/ptscotchDecomp/Make/options     |  1 +
 .../decompose/ptscotchDecomp/ptscotchDecomp.C |  9 ++++--
 src/renumber/zoltanRenumber/Make/options      |  1 +
 src/renumber/zoltanRenumber/zoltanRenumber.C  |  4 +--
 8 files changed, 53 insertions(+), 27 deletions(-)

diff --git a/src/Pstream/mpi/PstreamGlobals.C b/src/Pstream/mpi/PstreamGlobals.C
index 1acbe583f..8ca8c1720 100644
--- a/src/Pstream/mpi/PstreamGlobals.C
+++ b/src/Pstream/mpi/PstreamGlobals.C
@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2013-2015 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2013-2018 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -32,6 +32,8 @@ namespace Foam
 
 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
 
+MPI_Comm PstreamGlobals::MPI_COMM_FOAM;
+
 // Outstanding non-blocking operations.
 //! \cond fileScope
 DynamicList<MPI_Request> PstreamGlobals::outstandingRequests_;
diff --git a/src/Pstream/mpi/PstreamGlobals.H b/src/Pstream/mpi/PstreamGlobals.H
index f87a55b0c..e029816cb 100644
--- a/src/Pstream/mpi/PstreamGlobals.H
+++ b/src/Pstream/mpi/PstreamGlobals.H
@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2013-2015 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2013-2018 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -51,23 +51,20 @@ namespace Foam
 
 namespace PstreamGlobals
 {
+    extern MPI_Comm MPI_COMM_FOAM;
 
-extern DynamicList<MPI_Request> outstandingRequests_;
+    extern DynamicList<MPI_Request> outstandingRequests_;
 
-//extern int nRequests_;
-//extern DynamicList<label> freedRequests_;
+    extern int nTags_;
 
-extern int nTags_;
+    extern DynamicList<int> freedTags_;
 
-extern DynamicList<int> freedTags_;
+    // Current communicators. First element will be MPI_COMM_FOAM
+    extern DynamicList<MPI_Comm> MPICommunicators_;
 
+    extern DynamicList<MPI_Group> MPIGroups_;
 
-// Current communicators. First element will be MPI_COMM_WORLD
-extern DynamicList<MPI_Comm> MPICommunicators_;
-extern DynamicList<MPI_Group> MPIGroups_;
-
-void checkCommunicator(const label, const label procNo);
-
+    void checkCommunicator(const label, const label procNo);
 };
 
 
diff --git a/src/Pstream/mpi/UOPwrite.C b/src/Pstream/mpi/UOPwrite.C
index e0b226439..432e4bf2f 100644
--- a/src/Pstream/mpi/UOPwrite.C
+++ b/src/Pstream/mpi/UOPwrite.C
@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2017 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2018 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -77,7 +77,7 @@ bool Foam::UOPstream::write
             MPI_BYTE,
             toProcNo,   //procID(toProcNo),
             tag,
-            PstreamGlobals::MPICommunicators_[communicator] //MPI_COMM_WORLD
+            PstreamGlobals::MPICommunicators_[communicator]
         );
 
         if (debug)
@@ -97,7 +97,7 @@ bool Foam::UOPstream::write
             MPI_BYTE,
             toProcNo,   //procID(toProcNo),
             tag,
-            PstreamGlobals::MPICommunicators_[communicator] //MPI_COMM_WORLD
+            PstreamGlobals::MPICommunicators_[communicator]
         );
 
         if (debug)
@@ -119,7 +119,7 @@ bool Foam::UOPstream::write
             MPI_BYTE,
             toProcNo,   //procID(toProcNo),
             tag,
-            PstreamGlobals::MPICommunicators_[communicator],//MPI_COMM_WORLD,
+            PstreamGlobals::MPICommunicators_[communicator],
             &request
         );
 
diff --git a/src/Pstream/mpi/UPstream.C b/src/Pstream/mpi/UPstream.C
index ab231d59d..00ea4684f 100644
--- a/src/Pstream/mpi/UPstream.C
+++ b/src/Pstream/mpi/UPstream.C
@@ -77,10 +77,25 @@ bool Foam::UPstream::init(int& argc, char**& argv)
         &provided_thread_support
     );
 
+    // int numprocs;
+    // MPI_Comm_size(MPI_COMM_WORLD, &numprocs);
+    // int myRank;
+    // MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
+
+    int myGlobalRank;
+    MPI_Comm_rank(MPI_COMM_WORLD, &myGlobalRank);
+    MPI_Comm_split
+    (
+        MPI_COMM_WORLD,
+        1,
+        myGlobalRank,
+        &PstreamGlobals::MPI_COMM_FOAM
+    );
+
     int numprocs;
-    MPI_Comm_size(MPI_COMM_WORLD, &numprocs);
+    MPI_Comm_size(PstreamGlobals::MPI_COMM_FOAM, &numprocs);
     int myRank;
-    MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
+    MPI_Comm_rank(PstreamGlobals::MPI_COMM_FOAM, &myRank);
 
     if (debug)
     {
@@ -175,14 +190,14 @@ void Foam::UPstream::exit(int errnum)
     }
     else
     {
-        MPI_Abort(MPI_COMM_WORLD, errnum);
+        MPI_Abort(PstreamGlobals::MPI_COMM_FOAM, errnum);
     }
 }
 
 
 void Foam::UPstream::abort()
 {
-    MPI_Abort(MPI_COMM_WORLD, 1);
+    MPI_Abort(PstreamGlobals::MPI_COMM_FOAM, 1);
 }
 
 
@@ -583,8 +598,13 @@ void Foam::UPstream::allocatePstreamCommunicator
                 << UPstream::worldComm << Foam::exit(FatalError);
         }
 
-        PstreamGlobals::MPICommunicators_[index] = MPI_COMM_WORLD;
-        MPI_Comm_group(MPI_COMM_WORLD, &PstreamGlobals::MPIGroups_[index]);
+        PstreamGlobals::MPICommunicators_[index] =
+            PstreamGlobals::MPI_COMM_FOAM;
+        MPI_Comm_group
+        (
+            PstreamGlobals::MPI_COMM_FOAM,
+            &PstreamGlobals::MPIGroups_[index]
+        );
         MPI_Comm_rank
         (
             PstreamGlobals::MPICommunicators_[index],
diff --git a/src/parallel/decompose/ptscotchDecomp/Make/options b/src/parallel/decompose/ptscotchDecomp/Make/options
index cb407ec6b..5e3069ba8 100644
--- a/src/parallel/decompose/ptscotchDecomp/Make/options
+++ b/src/parallel/decompose/ptscotchDecomp/Make/options
@@ -3,6 +3,7 @@ sinclude $(RULES)/mplib$(WM_MPLIB)
 
 EXE_INC = \
     $(PFLAGS) $(PINC) \
+    -I$(FOAM_SRC)/Pstream/mpi/lnInclude \
     -I$(SCOTCH_ARCH_PATH)/include/$(FOAM_MPI) \
     -I$(SCOTCH_ARCH_PATH)/include \
     -I/usr/include/scotch \
diff --git a/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C b/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C
index 10a1a93ec..c1c0c0c62 100644
--- a/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C
+++ b/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C
@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2017 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2018 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -29,6 +29,7 @@ License
 #include "OFstream.H"
 #include "globalIndex.H"
 #include "SubField.H"
+#include "PstreamGlobals.H"
 
 extern "C"
 {
@@ -468,7 +469,11 @@ Foam::label Foam::ptscotchDecomp::decompose
         Pout<< "SCOTCH_dgraphInit" << endl;
     }
     SCOTCH_Dgraph grafdat;
-    check(SCOTCH_dgraphInit(&grafdat, MPI_COMM_WORLD), "SCOTCH_dgraphInit");
+    check
+    (
+        SCOTCH_dgraphInit(&grafdat, PstreamGlobals::MPI_COMM_FOAM),
+        "SCOTCH_dgraphInit"
+    );
 
 
     if (debug)
diff --git a/src/renumber/zoltanRenumber/Make/options b/src/renumber/zoltanRenumber/Make/options
index c637f39ba..45a153b37 100644
--- a/src/renumber/zoltanRenumber/Make/options
+++ b/src/renumber/zoltanRenumber/Make/options
@@ -4,6 +4,7 @@ sinclude $(RULES)/mplib$(WM_MPLIB)
 EXE_INC = \
     /* -DFULLDEBUG -g -O0 */ \
     $(PFLAGS) $(PINC) \
+    -I$(FOAM_SRC)/Pstream/mpi/lnInclude \
     -I$(FOAM_SRC)/renumber/renumberMethods/lnInclude \
     -I$(ZOLTAN_ARCH_PATH)/include/ \
     -I$(LIB_SRC)/meshTools/lnInclude
diff --git a/src/renumber/zoltanRenumber/zoltanRenumber.C b/src/renumber/zoltanRenumber/zoltanRenumber.C
index b487d522e..ad28745cc 100644
--- a/src/renumber/zoltanRenumber/zoltanRenumber.C
+++ b/src/renumber/zoltanRenumber/zoltanRenumber.C
@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2017 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2018 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -294,7 +294,7 @@ Foam::labelList Foam::zoltanRenumber::renumber
             << "Failed initialising Zoltan" << exit(FatalError);
     }
 
-    struct Zoltan_Struct *zz = Zoltan_Create(MPI_COMM_WORLD);
+    struct Zoltan_Struct *zz = Zoltan_Create(PstreamGlobals::MPI_COMM_FOAM);
 
     polyMesh& mesh = const_cast<polyMesh&>(pMesh);