diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UIPstream.C b/src/OpenFOAM/db/IOstreams/Pstreams/UIPstream.C index 4e853274d8..24b2af4af2 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/UIPstream.C +++ b/src/OpenFOAM/db/IOstreams/Pstreams/UIPstream.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011-2012 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -330,7 +330,7 @@ Foam::Istream& Foam::UIPstream::rewind() void Foam::UIPstream::print(Ostream& os) const { os << "Reading from processor " << fromProcNo_ - << " to processor " << myProcNo << " using communicator " << comm_ + << " using communicator " << comm_ << " and tag " << tag_ << Foam::endl; } diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C index ab0560cd09..9c49b28963 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C +++ b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C @@ -339,6 +339,39 @@ void Foam::UPstream::freeCommunicators(const bool doPstream) } +int Foam::UPstream::baseProcNo(const label myComm, const int myProcID) +{ + int procID = myProcID; + label comm = myComm; + + while (parent(comm) != -1) + { + const List& parentRanks = UPstream::procID(comm); + procID = parentRanks[procID]; + comm = UPstream::parent(comm); + } + + return procID; +} + + +Foam::label Foam::UPstream::myProcNo(const label myComm, const int baseProcID) +{ + const List& parentRanks = procID(myComm); + label parentComm = parent(myComm); + + if (parentComm == -1) + { + return findIndex(parentRanks, baseProcID); + } + else + { + label parentRank = myProcNo(parentComm, baseProcID); + return findIndex(parentRanks, parentRank); + } +} + + // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * // // By default this is not a parallel run diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H index 89f49f9509..ab6517dadb 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H @@ -323,18 +323,6 @@ public: comm_(allocateCommunicator(parent, subRanks, doPstream)) {} -// communicator(const label parent) -// : -// comm_ -// ( -// allocateCommunicator -// ( -// parent, -// identity(UPstream::nProcs(parent)) -// ) -// ) -// {} - ~communicator() { freeCommunicator(comm_); @@ -346,6 +334,14 @@ public: } }; + //- Return physical processor number (i.e. processor number in + // worldComm) given communicator and procssor + static int baseProcNo(const label myComm, const int myProcID); + + //- Return processor number in communicator (given physical processor + // number) (= reverse of baseProcNo) + static label myProcNo(const label myComm, const int baseProcID); + //- Add the valid option this type of communications library // adds/requires on the command line @@ -380,6 +376,12 @@ public: static void freeTag(const word&, const int tag); +// //- Return (index of) request to use in calls to nonblocking ops +// static label allocateRequest(const word&); +// +// //- Free (index of) request +// static void freeRequest(const word&, const label request); + //- Is this a parallel run? static bool& parRun() @@ -416,7 +418,6 @@ public: return parentCommunicator_(communicator); } - // //- Process IDs // static const List& procIDs() // { diff --git a/src/OpenFOAM/db/error/messageStream.C b/src/OpenFOAM/db/error/messageStream.C index 2aabacd9ba..402bede540 100644 --- a/src/OpenFOAM/db/error/messageStream.C +++ b/src/OpenFOAM/db/error/messageStream.C @@ -196,14 +196,7 @@ Foam::OSstream& Foam::messageStream::operator()(const label communicator) { if (title().size()) { - if (Pstream::parRun() && !collect) - { - Pout<< title().c_str(); - } - else - { - Sout<< title().c_str(); - } + Pout<< title().c_str(); } if (maxErrors_) @@ -218,14 +211,7 @@ Foam::OSstream& Foam::messageStream::operator()(const label communicator) } } - if (Pstream::parRun() && !collect) - { - return Pout; - } - else - { - return Sout; - } + return Pout; } } } diff --git a/src/OpenFOAM/matrices/LUscalarMatrix/LUscalarMatrix.C b/src/OpenFOAM/matrices/LUscalarMatrix/LUscalarMatrix.C index 0829e5fb72..be850e59b0 100644 --- a/src/OpenFOAM/matrices/LUscalarMatrix/LUscalarMatrix.C +++ b/src/OpenFOAM/matrices/LUscalarMatrix/LUscalarMatrix.C @@ -33,8 +33,8 @@ License Foam::LUscalarMatrix::LUscalarMatrix(const scalarSquareMatrix& matrix) : - comm_(Pstream::worldComm), scalarSquareMatrix(matrix), + comm_(Pstream::worldComm), pivotIndices_(n()) { LUDecompose(*this, pivotIndices_); @@ -67,11 +67,6 @@ Foam::LUscalarMatrix::LUscalarMatrix ) ); - -Pout<< "LUscalarMatrix :" - << " comm:" << comm_ - << " master:" << Pstream::master(comm_) << endl; - if (Pstream::master(comm_)) { for @@ -81,8 +76,6 @@ Pout<< "LUscalarMatrix :" slave++ ) { -Pout<< "Receiving from " << slave - << " using comm:" << comm_ << endl; lduMatrices.set ( lduMatrixi++, diff --git a/src/OpenFOAM/meshes/lduMesh/lduPrimitiveMesh.H b/src/OpenFOAM/meshes/lduMesh/lduPrimitiveMesh.H index b39d184927..8f562407b4 100644 --- a/src/OpenFOAM/meshes/lduMesh/lduPrimitiveMesh.H +++ b/src/OpenFOAM/meshes/lduMesh/lduPrimitiveMesh.H @@ -108,7 +108,7 @@ public: Pout<< "lduPrimitiveMesh :" << " nCells:" << nCells << " l:" << lowerAddr_.size() - << " u::" << upperAddr_.size() + << " u:" << upperAddr_.size() << " pa:" << patchAddr_.size() << " interfaces:" << interfaces_.size() << " comm:" << comm_ @@ -124,9 +124,11 @@ public: const processorLduInterface >(interfaces_[i]); - Pout<< " comm:" << pi.comm() + Pout<< " patch:" << i + << " size:" << patchAddr_[i].size() << " myProcNo:" << pi.myProcNo() << " neighbProcNo:" << pi.neighbProcNo() + << " comm:" << pi.comm() << endl; } } @@ -158,7 +160,7 @@ public: Pout<< "lduPrimitiveMesh :" << " nCells:" << nCells << " l:" << lowerAddr_.size() - << " u::" << upperAddr_.size() + << " u:" << upperAddr_.size() << " pa:" << patchAddr_.size() << " interfaces:" << interfaces_.size() << " comm:" << comm_ @@ -174,9 +176,11 @@ public: const processorLduInterface >(interfaces_[i]); - Pout<< " comm:" << pi.comm() + Pout<< " patch:" << i + << " size:" << patchAddr_[i].size() << " myProcNo:" << pi.myProcNo() << " neighbProcNo:" << pi.neighbProcNo() + << " comm:" << pi.comm() << endl; } } diff --git a/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalIndex.C b/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalIndex.C index 8ecd6ab8f9..3d6a6fc025 100644 --- a/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalIndex.C +++ b/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalIndex.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -31,18 +31,51 @@ Foam::globalIndex::globalIndex ( const label localSize, const int tag, + const label comm, const bool parallel ) +: + offsets_(Pstream::nProcs(comm)+1) +{ + labelList localSizes(Pstream::nProcs(comm), 0); + localSizes[Pstream::myProcNo(comm)] = localSize; + if (parallel) + { + Pstream::gatherList(localSizes, tag, comm); + Pstream::scatterList(localSizes, tag, comm); + } + + label offset = 0; + offsets_[0] = 0; + for (label procI = 0; procI < Pstream::nProcs(comm); procI++) + { + label oldOffset = offset; + offset += localSizes[procI]; + + if (offset < oldOffset) + { + FatalErrorIn + ( + "globalIndex::globalIndex" + "(const label, const int, const label, const bool)" + ) << "Overflow : sum of sizes " << localSizes + << " exceeds capability of label (" << labelMax + << "). Please recompile with larger datatype for label." + << exit(FatalError); + } + offsets_[procI+1] = offset; + } +} + + +Foam::globalIndex::globalIndex(const label localSize) : offsets_(Pstream::nProcs()+1) { labelList localSizes(Pstream::nProcs(), 0); localSizes[Pstream::myProcNo()] = localSize; - if (parallel) - { - Pstream::gatherList(localSizes, tag); - Pstream::scatterList(localSizes, tag); - } + Pstream::gatherList(localSizes, Pstream::msgType()); + Pstream::scatterList(localSizes, Pstream::msgType()); label offset = 0; offsets_[0] = 0; @@ -64,6 +97,12 @@ Foam::globalIndex::globalIndex } +Foam::globalIndex::globalIndex(const labelList& offsets) +: + offsets_(offsets) +{} + + Foam::globalIndex::globalIndex(Istream& is) { is >> offsets_; diff --git a/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalIndex.H b/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalIndex.H index c712066f4a..acfebc54d1 100644 --- a/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalIndex.H +++ b/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalIndex.H @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -71,21 +71,29 @@ public: // Constructors + + //- Construct from local max size + globalIndex(const label localSize); + //- Construct from local max size globalIndex ( const label localSize, - const int tag = Pstream::msgType(), - const bool parallel = true // use parallel comms + const int tag, + const label comm, + const bool parallel // use parallel comms ); + //- Construct from components + globalIndex(const labelList& offsets); + //- Construct from Istream globalIndex(Istream& is); // Member Functions - // Queries relating to my processor + // Queries relating to my processor (using world communicator) //- my local size inline label localSize() const; diff --git a/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalIndexI.H b/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalIndexI.H index b8261b3d0e..d85816f1b4 100644 --- a/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalIndexI.H +++ b/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalIndexI.H @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -47,7 +47,7 @@ inline Foam::label Foam::globalIndex::localSize() const inline Foam::label Foam::globalIndex::size() const { - return offsets_[Pstream::nProcs()]; + return offsets_.last(); } @@ -104,7 +104,7 @@ inline Foam::label Foam::globalIndex::toLocal(const label i) const inline Foam::label Foam::globalIndex::whichProcID(const label i) const { - if (i < 0 || i >= offsets_[Pstream::nProcs()]) + if (i < 0 || i >= offsets_.size()-1) { FatalErrorIn("globalIndex::whichProcID(const label)") << "Global " << i << " does not belong on any processor." diff --git a/src/OpenFOAM/meshes/polyMesh/polyPatches/constraint/processor/processorPolyPatch.C b/src/OpenFOAM/meshes/polyMesh/polyPatches/constraint/processor/processorPolyPatch.C index ab44c8d45b..515a78bbb8 100644 --- a/src/OpenFOAM/meshes/polyMesh/polyPatches/constraint/processor/processorPolyPatch.C +++ b/src/OpenFOAM/meshes/polyMesh/polyPatches/constraint/processor/processorPolyPatch.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011-2012 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -61,7 +61,7 @@ Foam::processorPolyPatch::processorPolyPatch ) : coupledPolyPatch(name, size, start, index, bm, typeName, transform), - comm_(comm), +// comm_(comm), myProcNo_(myProcNo), neighbProcNo_(neighbProcNo), neighbFaceCentres_(), @@ -80,10 +80,10 @@ Foam::processorPolyPatch::processorPolyPatch ) : coupledPolyPatch(name, dict, index, bm, patchType), - comm_ - ( - dict.lookupOrDefault("communicator", UPstream::worldComm) - ), +// comm_ +// ( +// dict.lookupOrDefault("communicator", UPstream::worldComm) +// ), myProcNo_(readLabel(dict.lookup("myProcNo"))), neighbProcNo_(readLabel(dict.lookup("neighbProcNo"))), neighbFaceCentres_(), @@ -99,7 +99,7 @@ Foam::processorPolyPatch::processorPolyPatch ) : coupledPolyPatch(pp, bm), - comm_(pp.comm_), +// comm_(pp.comm_), myProcNo_(pp.myProcNo_), neighbProcNo_(pp.neighbProcNo_), neighbFaceCentres_(), @@ -118,7 +118,7 @@ Foam::processorPolyPatch::processorPolyPatch ) : coupledPolyPatch(pp, bm, index, newSize, newStart), - comm_(pp.comm_), +// comm_(pp.comm_), myProcNo_(pp.myProcNo_), neighbProcNo_(pp.neighbProcNo_), neighbFaceCentres_(), @@ -137,7 +137,7 @@ Foam::processorPolyPatch::processorPolyPatch ) : coupledPolyPatch(pp, bm, index, mapAddressing, newStart), - comm_(pp.comm_), +// comm_(pp.comm_), myProcNo_(pp.myProcNo_), neighbProcNo_(pp.neighbProcNo_), neighbFaceCentres_(), @@ -1091,11 +1091,11 @@ bool Foam::processorPolyPatch::order void Foam::processorPolyPatch::write(Ostream& os) const { coupledPolyPatch::write(os); - if (comm_ != UPstream::worldComm) - { - os.writeKeyword("communicator") << comm_ - << token::END_STATEMENT << nl; - } +// if (comm_ != UPstream::worldComm) +// { +// os.writeKeyword("communicator") << comm_ +// << token::END_STATEMENT << nl; +// } os.writeKeyword("myProcNo") << myProcNo_ << token::END_STATEMENT << nl; os.writeKeyword("neighbProcNo") << neighbProcNo_ diff --git a/src/OpenFOAM/meshes/polyMesh/polyPatches/constraint/processor/processorPolyPatch.H b/src/OpenFOAM/meshes/polyMesh/polyPatches/constraint/processor/processorPolyPatch.H index 3156bf6af7..3bd72a7cf2 100644 --- a/src/OpenFOAM/meshes/polyMesh/polyPatches/constraint/processor/processorPolyPatch.H +++ b/src/OpenFOAM/meshes/polyMesh/polyPatches/constraint/processor/processorPolyPatch.H @@ -42,6 +42,7 @@ SourceFiles #include "coupledPolyPatch.H" #include "polyBoundaryMesh.H" #include "faceListFwd.H" +#include "polyMesh.H" // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // @@ -58,9 +59,6 @@ class processorPolyPatch { // Private data - //- Communicator to use - label comm_; - int myProcNo_; int neighbProcNo_; @@ -252,7 +250,8 @@ public: //- Return communicator used for communication label comm() const { - return comm_; + //return comm_; + return boundaryMesh().mesh().comm(); } //- Return processor number @@ -270,11 +269,11 @@ public: // For testing - //- Return communicator used for communication - label& comm() - { - return comm_; - } +// //- Return communicator used for communication +// label& comm() +// { +// return comm_; +// } //- Return processor number int& myProcNo() diff --git a/src/Pstream/mpi/PstreamGlobals.C b/src/Pstream/mpi/PstreamGlobals.C index c57e7aef24..cdafa043d7 100644 --- a/src/Pstream/mpi/PstreamGlobals.C +++ b/src/Pstream/mpi/PstreamGlobals.C @@ -37,6 +37,27 @@ namespace Foam DynamicList PstreamGlobals::outstandingRequests_; //! \endcond +//// Max outstanding non-blocking operations. +////! \cond fileScope +//int PstreamGlobals::nRequests_ = 0; +////! \endcond + +// Free'd non-blocking operations. +//! \cond fileScope +//DynamicList