PstreamGlobals::MPI_COMM_FOAM: New OpenFOAM specific MPI communicator

Splitting MPI_COMM_FOAM from MPI_COMM_WORLD allows OpenFOAM to be linked with
other libraries communicating via MPI.

Resolves feature request https://bugs.openfoam.org/view.php?id=2815

src/Pstream/mpi/PstreamGlobals.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2013-2015 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2013-2018 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -32,6 +32,8 @@ namespace Foam
 
 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
 
+MPI_Comm PstreamGlobals::MPI_COMM_FOAM;
+
 // Outstanding non-blocking operations.
 //! \cond fileScope
 DynamicList<MPI_Request> PstreamGlobals::outstandingRequests_;

src/Pstream/mpi/PstreamGlobals.H

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2013-2015 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2013-2018 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -51,23 +51,20 @@ namespace Foam
 
 namespace PstreamGlobals
 {
+    extern MPI_Comm MPI_COMM_FOAM;
 
-extern DynamicList<MPI_Request> outstandingRequests_;
+    extern DynamicList<MPI_Request> outstandingRequests_;
 
-//extern int nRequests_;
+    extern int nTags_;
-//extern DynamicList<label> freedRequests_;
 
-extern int nTags_;
+    extern DynamicList<int> freedTags_;
 
-extern DynamicList<int> freedTags_;
+    // Current communicators. First element will be MPI_COMM_FOAM
+    extern DynamicList<MPI_Comm> MPICommunicators_;
 
+    extern DynamicList<MPI_Group> MPIGroups_;
 
-// Current communicators. First element will be MPI_COMM_WORLD
+    void checkCommunicator(const label, const label procNo);
-extern DynamicList<MPI_Comm> MPICommunicators_;
-extern DynamicList<MPI_Group> MPIGroups_;
-
-void checkCommunicator(const label, const label procNo);
-
 };
 
 

src/Pstream/mpi/UOPwrite.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2017 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2018 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -77,7 +77,7 @@ bool Foam::UOPstream::write
             MPI_BYTE,
             toProcNo,   //procID(toProcNo),
             tag,
-            PstreamGlobals::MPICommunicators_[communicator] //MPI_COMM_WORLD
+            PstreamGlobals::MPICommunicators_[communicator]
         );
 
         if (debug)
@@ -97,7 +97,7 @@ bool Foam::UOPstream::write
             MPI_BYTE,
             toProcNo,   //procID(toProcNo),
             tag,
-            PstreamGlobals::MPICommunicators_[communicator] //MPI_COMM_WORLD
+            PstreamGlobals::MPICommunicators_[communicator]
         );
 
         if (debug)
@@ -119,7 +119,7 @@ bool Foam::UOPstream::write
             MPI_BYTE,
             toProcNo,   //procID(toProcNo),
             tag,
-            PstreamGlobals::MPICommunicators_[communicator],//MPI_COMM_WORLD,
+            PstreamGlobals::MPICommunicators_[communicator],
             &request
         );
 

src/Pstream/mpi/UPstream.C

@@ -77,10 +77,25 @@ bool Foam::UPstream::init(int& argc, char**& argv)
         &provided_thread_support
     );
 
+    // int numprocs;
+    // MPI_Comm_size(MPI_COMM_WORLD, &numprocs);
+    // int myRank;
+    // MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
+
+    int myGlobalRank;
+    MPI_Comm_rank(MPI_COMM_WORLD, &myGlobalRank);
+    MPI_Comm_split
+    (
+        MPI_COMM_WORLD,
+        1,
+        myGlobalRank,
+        &PstreamGlobals::MPI_COMM_FOAM
+    );
+
     int numprocs;
-    MPI_Comm_size(MPI_COMM_WORLD, &numprocs);
+    MPI_Comm_size(PstreamGlobals::MPI_COMM_FOAM, &numprocs);
     int myRank;
-    MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
+    MPI_Comm_rank(PstreamGlobals::MPI_COMM_FOAM, &myRank);
 
     if (debug)
     {
@@ -175,14 +190,14 @@ void Foam::UPstream::exit(int errnum)
     }
     else
     {
-        MPI_Abort(MPI_COMM_WORLD, errnum);
+        MPI_Abort(PstreamGlobals::MPI_COMM_FOAM, errnum);
     }
 }
 
 
 void Foam::UPstream::abort()
 {
-    MPI_Abort(MPI_COMM_WORLD, 1);
+    MPI_Abort(PstreamGlobals::MPI_COMM_FOAM, 1);
 }
 
 
@@ -583,8 +598,13 @@ void Foam::UPstream::allocatePstreamCommunicator
                 << UPstream::worldComm << Foam::exit(FatalError);
         }
 
-        PstreamGlobals::MPICommunicators_[index] = MPI_COMM_WORLD;
+        PstreamGlobals::MPICommunicators_[index] =
-        MPI_Comm_group(MPI_COMM_WORLD, &PstreamGlobals::MPIGroups_[index]);
+            PstreamGlobals::MPI_COMM_FOAM;
+        MPI_Comm_group
+        (
+            PstreamGlobals::MPI_COMM_FOAM,
+            &PstreamGlobals::MPIGroups_[index]
+        );
         MPI_Comm_rank
         (
             PstreamGlobals::MPICommunicators_[index],

src/parallel/decompose/ptscotchDecomp/Make/options

@@ -3,6 +3,7 @@ sinclude $(RULES)/mplib$(WM_MPLIB)
 
 EXE_INC = \
     $(PFLAGS) $(PINC) \
+    -I$(FOAM_SRC)/Pstream/mpi/lnInclude \
     -I$(SCOTCH_ARCH_PATH)/include/$(FOAM_MPI) \
     -I$(SCOTCH_ARCH_PATH)/include \
     -I/usr/include/scotch \

src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2017 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2018 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -29,6 +29,7 @@ License
 #include "OFstream.H"
 #include "globalIndex.H"
 #include "SubField.H"
+#include "PstreamGlobals.H"
 
 extern "C"
 {
@@ -468,7 +469,11 @@ Foam::label Foam::ptscotchDecomp::decompose
         Pout<< "SCOTCH_dgraphInit" << endl;
     }
     SCOTCH_Dgraph grafdat;
-    check(SCOTCH_dgraphInit(&grafdat, MPI_COMM_WORLD), "SCOTCH_dgraphInit");
+    check
+    (
+        SCOTCH_dgraphInit(&grafdat, PstreamGlobals::MPI_COMM_FOAM),
+        "SCOTCH_dgraphInit"
+    );
 
 
     if (debug)

src/renumber/zoltanRenumber/Make/options

@@ -4,6 +4,7 @@ sinclude $(RULES)/mplib$(WM_MPLIB)
 EXE_INC = \
     /* -DFULLDEBUG -g -O0 */ \
     $(PFLAGS) $(PINC) \
+    -I$(FOAM_SRC)/Pstream/mpi/lnInclude \
     -I$(FOAM_SRC)/renumber/renumberMethods/lnInclude \
     -I$(ZOLTAN_ARCH_PATH)/include/ \
     -I$(LIB_SRC)/meshTools/lnInclude

src/renumber/zoltanRenumber/zoltanRenumber.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2017 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2018 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -294,7 +294,7 @@ Foam::labelList Foam::zoltanRenumber::renumber
             << "Failed initialising Zoltan" << exit(FatalError);
     }
 
-    struct Zoltan_Struct *zz = Zoltan_Create(MPI_COMM_WORLD);
+    struct Zoltan_Struct *zz = Zoltan_Create(PstreamGlobals::MPI_COMM_FOAM);
 
     polyMesh& mesh = const_cast<polyMesh&>(pMesh);