From ce282dfbbf26eafab8affe0211c2a7fc888fff92 Mon Sep 17 00:00:00 2001 From: Mark Olesen Date: Thu, 20 Apr 2023 12:54:49 +0200 Subject: [PATCH] ENH: correct the selfComm procNo relative to world communicator STYLE: qualify worldComm as UPstream instead Pstream STYLE: PPCG::gSumMagProd() as static function --- .../test/parallel-comm2/Test-parallel-comm2.C | 15 ++++++----- src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C | 10 ++++--- src/OpenFOAM/db/error/error.C | 4 +-- .../matrices/LUscalarMatrix/LUscalarMatrix.C | 26 +++++++++---------- .../matrices/lduMatrix/solvers/PPCG/PPCG.C | 6 ++--- .../matrices/lduMatrix/solvers/PPCG/PPCG.H | 4 +-- src/Pstream/mpi/UPstream.C | 2 +- src/finiteArea/faMesh/faMesh.C | 8 +++--- 8 files changed, 40 insertions(+), 35 deletions(-) diff --git a/applications/test/parallel-comm2/Test-parallel-comm2.C b/applications/test/parallel-comm2/Test-parallel-comm2.C index ce01dc7cf9..660bc6fb94 100644 --- a/applications/test/parallel-comm2/Test-parallel-comm2.C +++ b/applications/test/parallel-comm2/Test-parallel-comm2.C @@ -89,7 +89,7 @@ int main(int argc, char *argv[]) Info<< nl << "parallel:" << UPstream::parRun() - << "nProcs = " << UPstream::nProcs() + << " nProcs = " << UPstream::nProcs() << " with " << UPstream::nComms() << " predefined comm(s)." << " proc:" << UPstream::myProcNo() << nl; @@ -103,11 +103,14 @@ int main(int argc, char *argv[]) { Info<< nl; - //- Process IDs within a given communicator - Info<< "procIDs: " - << flatOutput(UPstream::procID(UPstream::commWorld())) << endl; - - rankInfo(UPstream::commWorld()); + // Process IDs within a given communicator + for (label comm = 0; comm < UPstream::nComms(); ++comm) + { + Info<< "comm=" << comm + << " procIDs: " << flatOutput(UPstream::procID(comm)) << endl; + rankInfo(comm); + Pout<< nl; + } Pout<< endl; } diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C index ae95f15daf..3d6dc38439 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C +++ b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C @@ -165,7 +165,7 @@ void Foam::UPstream::setParRun(const label nProcs, const bool haveThreads) // Failed sanity check FatalErrorInFunction << "problem : comm:" << comm - << " UPstream::globalComm:" << UPstream::globalComm + << " expected globalComm:" << UPstream::globalComm << Foam::exit(FatalError); } @@ -176,7 +176,7 @@ void Foam::UPstream::setParRun(const label nProcs, const bool haveThreads) // Failed sanity check FatalErrorInFunction << "problem : comm:" << comm - << " UPstream::selfComm:" << UPstream::selfComm + << " expected selfComm:" << UPstream::selfComm << Foam::exit(FatalError); } @@ -199,18 +199,20 @@ void Foam::UPstream::setParRun(const label nProcs, const bool haveThreads) // Failed sanity check FatalErrorInFunction << "problem : comm:" << comm - << " UPstream::globalComm:" << UPstream::globalComm + << " expected globalComm:" << UPstream::globalComm << Foam::exit(FatalError); } // 1: selfComm + // - Processor number wrt world communicator + singleProc.front() = myProcNo(UPstream::globalComm); comm = allocateCommunicator(-2, singleProc, true); if (comm != UPstream::selfComm) { // Failed sanity check FatalErrorInFunction << "problem : comm:" << comm - << " UPstream::selfComm:" << UPstream::selfComm + << " expected selfComm:" << UPstream::selfComm << Foam::exit(FatalError); } diff --git a/src/OpenFOAM/db/error/error.C b/src/OpenFOAM/db/error/error.C index 4adfae5f42..5dce9fd6cd 100644 --- a/src/OpenFOAM/db/error/error.C +++ b/src/OpenFOAM/db/error/error.C @@ -33,7 +33,7 @@ Note #include "fileName.H" #include "dictionary.H" #include "JobInfo.H" -#include "Pstream.H" +#include "UPstream.H" #include "StringStream.H" #include "foamVersion.H" #include "OSspecific.H" @@ -44,7 +44,7 @@ Note bool Foam::error::master(const label communicator) { // Trap negative value for comm as 'default'. This avoids direct use - // of Pstream::worldComm which may not have been initialised + // of UPstream::worldComm which may have not yet been initialised return ( diff --git a/src/OpenFOAM/matrices/LUscalarMatrix/LUscalarMatrix.C b/src/OpenFOAM/matrices/LUscalarMatrix/LUscalarMatrix.C index 65468a617c..2d03159abe 100644 --- a/src/OpenFOAM/matrices/LUscalarMatrix/LUscalarMatrix.C +++ b/src/OpenFOAM/matrices/LUscalarMatrix/LUscalarMatrix.C @@ -44,14 +44,14 @@ namespace Foam Foam::LUscalarMatrix::LUscalarMatrix() : - comm_(Pstream::worldComm) + comm_(UPstream::worldComm) {} Foam::LUscalarMatrix::LUscalarMatrix(const scalarSquareMatrix& matrix) : scalarSquareMatrix(matrix), - comm_(Pstream::worldComm), + comm_(UPstream::worldComm), pivotIndices_(m()) { LUDecompose(*this, pivotIndices_); @@ -67,9 +67,9 @@ Foam::LUscalarMatrix::LUscalarMatrix : comm_(ldum.mesh().comm()) { - if (Pstream::parRun()) + if (UPstream::parRun()) { - PtrList lduMatrices(Pstream::nProcs(comm_)); + PtrList lduMatrices(UPstream::nProcs(comm_)); label lduMatrixi = 0; @@ -84,9 +84,9 @@ Foam::LUscalarMatrix::LUscalarMatrix ) ); - if (Pstream::master(comm_)) + if (UPstream::master(comm_)) { - for (const int proci : Pstream::subProcs(comm_)) + for (const int proci : UPstream::subProcs(comm_)) { lduMatrices.set ( @@ -95,10 +95,10 @@ Foam::LUscalarMatrix::LUscalarMatrix ( IPstream ( - Pstream::commsTypes::scheduled, + UPstream::commsTypes::scheduled, proci, 0, // bufSize - Pstream::msgType(), + UPstream::msgType(), comm_ )() ) @@ -109,10 +109,10 @@ Foam::LUscalarMatrix::LUscalarMatrix { OPstream toMaster ( - Pstream::commsTypes::scheduled, - Pstream::masterNo(), + UPstream::commsTypes::scheduled, + UPstream::masterNo(), 0, // bufSize - Pstream::msgType(), + UPstream::msgType(), comm_ ); procLduMatrix cldum @@ -125,7 +125,7 @@ Foam::LUscalarMatrix::LUscalarMatrix } - if (Pstream::master(comm_)) + if (UPstream::master(comm_)) { label nCells = 0; forAll(lduMatrices, i) @@ -146,7 +146,7 @@ Foam::LUscalarMatrix::LUscalarMatrix convert(ldum, interfaceCoeffs, interfaces); } - if (Pstream::master(comm_)) + if (UPstream::master(comm_)) { if (debug) { diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/PPCG/PPCG.C b/src/OpenFOAM/matrices/lduMatrix/solvers/PPCG/PPCG.C index 53c0900de0..ea88f04462 100644 --- a/src/OpenFOAM/matrices/lduMatrix/solvers/PPCG/PPCG.C +++ b/src/OpenFOAM/matrices/lduMatrix/solvers/PPCG/PPCG.C @@ -51,7 +51,7 @@ void Foam::PPCG::gSumMagProd const solveScalarField& sumMag, UPstream::Request& request, const label comm -) const +) { const label nCells = a.size(); @@ -63,14 +63,14 @@ void Foam::PPCG::gSumMagProd globalSum[2] += mag(sumMag[cell]); } - if (Pstream::parRun()) + if (UPstream::parRun()) { Foam::reduce ( globalSum.data(), globalSum.size(), sumOp(), - Pstream::msgType(), + UPstream::msgType(), // (ignored): direct MPI call comm, request ); diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/PPCG/PPCG.H b/src/OpenFOAM/matrices/lduMatrix/solvers/PPCG/PPCG.H index a15bdf8ca2..964089a6d7 100644 --- a/src/OpenFOAM/matrices/lduMatrix/solvers/PPCG/PPCG.H +++ b/src/OpenFOAM/matrices/lduMatrix/solvers/PPCG/PPCG.H @@ -69,7 +69,7 @@ class PPCG // Private Member Functions //- Non-blocking version of sum(a*b), sum(a*c), sum(mag(sumMag)) - void gSumMagProd + static void gSumMagProd ( FixedList& globalSum, const solveScalarField& a, @@ -78,7 +78,7 @@ class PPCG const solveScalarField& sumMag, UPstream::Request& request, const label comm - ) const; + ); //- No copy construct PPCG(const PPCG&) = delete; diff --git a/src/Pstream/mpi/UPstream.C b/src/Pstream/mpi/UPstream.C index b980948228..67acb35cbf 100644 --- a/src/Pstream/mpi/UPstream.C +++ b/src/Pstream/mpi/UPstream.C @@ -299,7 +299,7 @@ bool Foam::UPstream::init(int& argc, char**& argv, const bool needsThread) Pstream::gatherList(worlds, UPstream::msgType(), UPstream::globalComm); // Compact - if (Pstream::master(UPstream::globalComm)) + if (UPstream::master(UPstream::globalComm)) { DynamicList worldNames(numprocs); worldIDs_.resize_nocopy(numprocs); diff --git a/src/finiteArea/faMesh/faMesh.C b/src/finiteArea/faMesh/faMesh.C index cf953b574c..b38b19ee3d 100644 --- a/src/finiteArea/faMesh/faMesh.C +++ b/src/finiteArea/faMesh/faMesh.C @@ -306,7 +306,7 @@ bool Foam::faMesh::init(const bool doInit) } // Create global mesh data - if (Pstream::parRun()) + if (UPstream::parRun()) { (void)globalData(); } @@ -378,7 +378,7 @@ Foam::faMesh::faMesh ), *this ), - comm_(Pstream::worldComm), + comm_(UPstream::worldComm), curTimeIndex_(time().timeIndex()), patchPtr_(nullptr), @@ -484,7 +484,7 @@ Foam::faMesh::faMesh *this, label(0) ), - comm_(Pstream::worldComm), + comm_(UPstream::worldComm), curTimeIndex_(time().timeIndex()), patchPtr_(nullptr), @@ -565,7 +565,7 @@ Foam::faMesh::faMesh *this, label(0) ), - comm_(Pstream::worldComm), + comm_(UPstream::worldComm), curTimeIndex_(time().timeIndex()), patchPtr_(nullptr),