zhyang-dev/openfoam
1
0
Fork 0
mirror of https://develop.openfoam.com/Development/openfoam.git synced 2025-11-28 03:28:01 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

ENH: cleaner separation of global and local world naming

Browse Source 
- UPstream::globalComm constant always refers to MPI_COMM_WORLD but
  UPstream::worldComm could be MPI_COMM_WORLD (single world)
  or a dedicated local communicator (for multi-world).

- provide a Pstream wrapped version of MPI_COMM_SELF,
  references as UPstream::selfComm

- UPstream::isUserComm(label)
  test for additional user-defined communicators
This commit is contained in:
Mark Olesen
2022-11-29 11:37:29 +01:00
committed by Andrew Heather
parent 7fe8bdcf99
commit ffeef76d8f
14 changed files with 575 additions and 229 deletions
Download Patch File Download Diff File Expand all files Collapse all files

102
applications/test/parallel-external-init/Test-parallel-external-init.C
View File

@ -5,7 +5,7 @@
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2019 OpenCFD Ltd.
Copyright (C) 2019-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -47,16 +47,88 @@ using namespace Foam;
bool startMPI()
{
int nprocs = 0, rank = 0;
enum whichComm : int { worldComm = 0, selfComm, nullComm };
int nprocs[3];
int rank[3];
int group_nprocs[3];
int group_rank[3];
MPI_Group mpiGroup;
MPI_Init(nullptr, nullptr);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs[worldComm]);
MPI_Comm_rank(MPI_COMM_WORLD, &rank[worldComm]);
if (nprocs && rank == 0)
const bool isMaster = (rank[worldComm] == 0);
const string prefix = '[' + Foam::name(rank[worldComm]) + "] ";
MPI_Comm_group(MPI_COMM_WORLD, &mpiGroup);
MPI_Group_size(mpiGroup, &group_nprocs[worldComm]);
MPI_Group_rank(mpiGroup, &group_rank[worldComm]);
if (isMaster && nprocs[worldComm])
{
std::cout<< nl << "Using MPI with " << nprocs << " procs" << nl << nl;
std::cout
<< nl << "Using MPI with " << nprocs[worldComm]
<< " procs, group:"
<< group_nprocs[worldComm] << nl
<< "World group: " << Foam::name(mpiGroup) << nl
<< nl;
}
MPI_Comm worldMpiComm;
MPI_Comm_dup(MPI_COMM_WORLD, &worldMpiComm);
MPI_Comm_group(MPI_COMM_WORLD, &mpiGroup);
if (isMaster && nprocs[worldComm])
{
std::cout
<< "dup comm group: " << Foam::name(mpiGroup) << nl;
}
MPI_Comm_free(&worldMpiComm);
// May be a bad idea
MPI_Group_free(&mpiGroup);
MPI_Comm_size(MPI_COMM_SELF, &nprocs[selfComm]);
MPI_Comm_rank(MPI_COMM_SELF, &rank[selfComm]);
MPI_Comm_group(MPI_COMM_SELF, &mpiGroup);
MPI_Group_size(mpiGroup, &group_nprocs[selfComm]);
MPI_Group_rank(mpiGroup, &group_rank[selfComm]);
if (isMaster && nprocs[worldComm])
{
std::cout
<< nl
<< "Self group: " << Foam::name(mpiGroup) << nl;
}
// Should be a bad idea
MPI_Group_free(&mpiGroup);
// if (nprocs && isMaster)
{
std::cout
<< prefix
<< "Self: " << rank[selfComm] << " from " << nprocs[selfComm]
<< " procs, group:"
<< group_nprocs[selfComm] << nl;
}
if (isMaster)
{
std::cout
<< "MPI_COMM_NULL: " << MPI_COMM_NULL << nl
<< "MPI_COMM_SELF: " << MPI_COMM_SELF << nl
<< "MPI_COMM_WORLD: " << MPI_COMM_WORLD << nl;
}
return true;
@ -77,8 +149,8 @@ string message()
{
return
(
"rank " + name(Pstream::myProcNo())
+ " / " + name(Pstream::nProcs()) + "\n"
"rank " + Foam::name(Pstream::myProcNo())
+ " / " + Foam::name(Pstream::nProcs()) + "\n"
);
}
@ -87,9 +159,21 @@ string message()
int main(int argc, char *argv[])
{
argList::noBanner();
argList::noCheckProcessorDirectories();
argList::addBoolOption("verbose", "Set debug level");
UPstream::debug = 1;
// Need to capture manually, since we need values before proper startup
int nVerbose = 0;
for (int argi = 1; argi < argc; ++argi)
{
if (strcmp(argv[argi], "-verbose") == 0)
{
++nVerbose;
}
}
UPstream::debug = nVerbose;
startMPI();