zhyang-dev/OpenFOAM-6
1
0
Fork 0
mirror of https://github.com/OpenFOAM/OpenFOAM-6.git synced 2025-12-08 06:57:46 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

PstreamGlobals::MPI_COMM_FOAM: New OpenFOAM specific MPI communicator

Browse Source 
Splitting MPI_COMM_FOAM from MPI_COMM_WORLD allows OpenFOAM to be linked with
other libraries communicating via MPI.

Resolves feature request https://bugs.openfoam.org/view.php?id=2815
This commit is contained in:
Henry Weller
2018-02-02 19:20:58 +00:00
parent be2261ca26
commit 73e9b2804f
8 changed files with 53 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/Pstream/mpi/PstreamGlobals.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2013-2015 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2013-2018 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -32,6 +32,8 @@ namespace Foam
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
MPI_Comm PstreamGlobals::MPI_COMM_FOAM;
// Outstanding non-blocking operations.
//! \cond fileScope
DynamicList<MPI_Request> PstreamGlobals::outstandingRequests_;

11
src/Pstream/mpi/PstreamGlobals.H
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2013-2015 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2013-2018 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -51,23 +51,20 @@ namespace Foam
namespace PstreamGlobals
{
extern MPI_Comm MPI_COMM_FOAM;
extern DynamicList<MPI_Request> outstandingRequests_;
//extern int nRequests_;
//extern DynamicList<label> freedRequests_;
extern int nTags_;
extern DynamicList<int> freedTags_;
// Current communicators. First element will be MPI_COMM_WORLD
// Current communicators. First element will be MPI_COMM_FOAM
extern DynamicList<MPI_Comm> MPICommunicators_;
extern DynamicList<MPI_Group> MPIGroups_;
void checkCommunicator(const label, const label procNo);
};

8
src/Pstream/mpi/UOPwrite.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2017 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2018 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -77,7 +77,7 @@ bool Foam::UOPstream::write
MPI_BYTE,
toProcNo, //procID(toProcNo),
tag,
PstreamGlobals::MPICommunicators_[communicator] //MPI_COMM_WORLD
PstreamGlobals::MPICommunicators_[communicator]
);
if (debug)
@ -97,7 +97,7 @@ bool Foam::UOPstream::write
MPI_BYTE,
toProcNo, //procID(toProcNo),
tag,
PstreamGlobals::MPICommunicators_[communicator] //MPI_COMM_WORLD
PstreamGlobals::MPICommunicators_[communicator]
);
if (debug)
@ -119,7 +119,7 @@ bool Foam::UOPstream::write
MPI_BYTE,
toProcNo, //procID(toProcNo),
tag,
PstreamGlobals::MPICommunicators_[communicator],//MPI_COMM_WORLD,
PstreamGlobals::MPICommunicators_[communicator],
&request
);

32
src/Pstream/mpi/UPstream.C
View File

@ -77,10 +77,25 @@ bool Foam::UPstream::init(int& argc, char**& argv)
&provided_thread_support
);
// int numprocs;
// MPI_Comm_size(MPI_COMM_WORLD, &numprocs);
// int myRank;
// MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
int myGlobalRank;
MPI_Comm_rank(MPI_COMM_WORLD, &myGlobalRank);
MPI_Comm_split
(
MPI_COMM_WORLD,
1,
myGlobalRank,
&PstreamGlobals::MPI_COMM_FOAM
);
int numprocs;
MPI_Comm_size(MPI_COMM_WORLD, &numprocs);
MPI_Comm_size(PstreamGlobals::MPI_COMM_FOAM, &numprocs);
int myRank;
MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
MPI_Comm_rank(PstreamGlobals::MPI_COMM_FOAM, &myRank);
if (debug)
{
@ -175,14 +190,14 @@ void Foam::UPstream::exit(int errnum)
}
else
{
MPI_Abort(MPI_COMM_WORLD, errnum);
MPI_Abort(PstreamGlobals::MPI_COMM_FOAM, errnum);
}
}
void Foam::UPstream::abort()
{
MPI_Abort(MPI_COMM_WORLD, 1);
MPI_Abort(PstreamGlobals::MPI_COMM_FOAM, 1);
}
@ -583,8 +598,13 @@ void Foam::UPstream::allocatePstreamCommunicator
<< UPstream::worldComm << Foam::exit(FatalError);
}
PstreamGlobals::MPICommunicators_[index] = MPI_COMM_WORLD;
MPI_Comm_group(MPI_COMM_WORLD, &PstreamGlobals::MPIGroups_[index]);
PstreamGlobals::MPICommunicators_[index] =
PstreamGlobals::MPI_COMM_FOAM;
MPI_Comm_group
(
PstreamGlobals::MPI_COMM_FOAM,
&PstreamGlobals::MPIGroups_[index]
);
MPI_Comm_rank
(
PstreamGlobals::MPICommunicators_[index],

1
src/parallel/decompose/ptscotchDecomp/Make/options
View File

@ -3,6 +3,7 @@ sinclude $(RULES)/mplib$(WM_MPLIB)
EXE_INC = \
$(PFLAGS) $(PINC) \
-I$(FOAM_SRC)/Pstream/mpi/lnInclude \
-I$(SCOTCH_ARCH_PATH)/include/$(FOAM_MPI) \
-I$(SCOTCH_ARCH_PATH)/include \
-I/usr/include/scotch \

9
src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2017 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2018 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -29,6 +29,7 @@ License
#include "OFstream.H"
#include "globalIndex.H"
#include "SubField.H"
#include "PstreamGlobals.H"
extern "C"
{
@ -468,7 +469,11 @@ Foam::label Foam::ptscotchDecomp::decompose
Pout<< "SCOTCH_dgraphInit" << endl;
}
SCOTCH_Dgraph grafdat;
check(SCOTCH_dgraphInit(&grafdat, MPI_COMM_WORLD), "SCOTCH_dgraphInit");
check
(
SCOTCH_dgraphInit(&grafdat, PstreamGlobals::MPI_COMM_FOAM),
"SCOTCH_dgraphInit"
);
if (debug)

1
src/renumber/zoltanRenumber/Make/options
View File

@ -4,6 +4,7 @@ sinclude $(RULES)/mplib$(WM_MPLIB)
EXE_INC = \
/* -DFULLDEBUG -g -O0 */ \
$(PFLAGS) $(PINC) \
-I$(FOAM_SRC)/Pstream/mpi/lnInclude \
-I$(FOAM_SRC)/renumber/renumberMethods/lnInclude \
-I$(ZOLTAN_ARCH_PATH)/include/ \
-I$(LIB_SRC)/meshTools/lnInclude

4
src/renumber/zoltanRenumber/zoltanRenumber.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2017 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2018 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -294,7 +294,7 @@ Foam::labelList Foam::zoltanRenumber::renumber
<< "Failed initialising Zoltan" << exit(FatalError);
}
struct Zoltan_Struct *zz = Zoltan_Create(MPI_COMM_WORLD);
struct Zoltan_Struct *zz = Zoltan_Create(PstreamGlobals::MPI_COMM_FOAM);
polyMesh& mesh = const_cast<polyMesh&>(pMesh);