diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H index dcf731b71b..acbd44c17d 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H @@ -351,12 +351,6 @@ public: //- Number of currently defined communicators static label nComms() noexcept { return parentComm_.size(); } - //- True if communicator appears to be user-allocated - static bool isUserComm(const label communicator) noexcept - { - return (communicator > worldComm && communicator > selfComm); - } - //- Debugging: print the communication tree static void printCommTree(const label communicator); diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.C b/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.C index 93eba9c1cf..3f4621f540 100644 --- a/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.C +++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.C @@ -335,10 +335,7 @@ Foam::OFstreamCollator::~OFstreamCollator() thread_.reset(nullptr); } - if (threadComm_ != -1) - { - UPstream::freeCommunicator(threadComm_); - } + UPstream::freeCommunicator(threadComm_); } diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C b/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C index e12779bf69..3ed78ec339 100644 --- a/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C +++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2017-2018 OpenFOAM Foundation - Copyright (C) 2020-2022 OpenCFD Ltd. + Copyright (C) 2020-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -287,7 +287,7 @@ Foam::fileOperations::collatedFileOperation::collatedFileOperation ), false ), - managedComm_(comm_), + managedComm_(getManagedComm(comm_)), // Possibly locally allocated writer_(mag(maxThreadFileBufferSize), comm_), nProcs_(Pstream::nProcs()), ioRanks_(ioRanks()) @@ -300,7 +300,6 @@ Foam::fileOperations::collatedFileOperation::collatedFileOperation ( const label comm, const labelList& ioRanks, - const word& typeName, bool verbose ) : @@ -314,6 +313,13 @@ Foam::fileOperations::collatedFileOperation::collatedFileOperation } +void Foam::fileOperations::collatedFileOperation::storeComm() const +{ + // From externally -> locally managed + managedComm_ = getManagedComm(comm_); +} + + // * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * // Foam::fileOperations::collatedFileOperation::~collatedFileOperation() @@ -321,10 +327,7 @@ Foam::fileOperations::collatedFileOperation::~collatedFileOperation() // Wait for any outstanding file operations flush(); - if (UPstream::isUserComm(managedComm_)) - { - UPstream::freeCommunicator(managedComm_); - } + UPstream::freeCommunicator(managedComm_); } diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.H b/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.H index fdfd59c2c4..836a105e21 100644 --- a/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.H +++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.H @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2017 OpenFOAM Foundation - Copyright (C) 2019-2021 OpenCFD Ltd. + Copyright (C) 2019-2022 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -46,8 +46,8 @@ SourceFiles \*---------------------------------------------------------------------------*/ -#ifndef fileOperations_collatedFileOperation_H -#define fileOperations_collatedFileOperation_H +#ifndef Foam_fileOperations_collatedFileOperation_H +#define Foam_fileOperations_collatedFileOperation_H #include "masterUncollatedFileOperation.H" #include "OFstreamCollator.H" @@ -70,7 +70,7 @@ class collatedFileOperation // Private Data //- Communicator allocated/managed by us - label managedComm_; + mutable label managedComm_; // Private Member Functions @@ -78,6 +78,7 @@ class collatedFileOperation //- Any initialisation steps after constructing void init(bool verbose); + protected: // Protected Data @@ -129,15 +130,14 @@ public: // Constructors //- Default construct - explicit collatedFileOperation(bool verbose); + explicit collatedFileOperation(bool verbose = false); //- Construct from user communicator collatedFileOperation ( const label comm, const labelList& ioRanks, - const word& typeName, - bool verbose + bool verbose = false ); @@ -147,6 +147,11 @@ public: // Member Functions + //- Transfer ownership of communicator to this fileOperation. + //- Use with caution + virtual void storeComm() const; + + // (reg)IOobject functionality //- Generate disk file name for object. Opposite of filePath. diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.C b/src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.C index f86e2c6ca3..223e54e0df 100644 --- a/src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.C +++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2017-2018 OpenFOAM Foundation - Copyright (C) 2021-2022 OpenCFD Ltd. + Copyright (C) 2021-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -144,26 +144,29 @@ Foam::fileOperations::hostCollatedFileOperation::hostCollatedFileOperation UPstream::allocateCommunicator ( UPstream::worldComm, - subRanks(Pstream::nProcs()) + subRanks(UPstream::nProcs()) ), - (Pstream::parRun() ? labelList() : ioRanks()), // processor dirs - typeName, - false // verbose + (UPstream::parRun() ? labelList() : ioRanks()), // processor dirs + false // verbose ), - managedComm_(comm_) + managedComm_(getManagedComm(comm_)) // Possibly locally allocated { init(verbose); } +void Foam::fileOperations::hostCollatedFileOperation::storeComm() const +{ + // From externally -> locally managed + managedComm_ = getManagedComm(comm_); +} + + // * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * // Foam::fileOperations::hostCollatedFileOperation::~hostCollatedFileOperation() { - if (UPstream::isUserComm(managedComm_)) - { - UPstream::freeCommunicator(managedComm_); - } + UPstream::freeCommunicator(managedComm_); } diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.H b/src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.H index 859d2705a1..333b3e0773 100644 --- a/src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.H +++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.H @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2017-2018 OpenFOAM Foundation - Copyright (C) 2021-2022 OpenCFD Ltd. + Copyright (C) 2021-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -57,8 +57,8 @@ SourceFiles \*---------------------------------------------------------------------------*/ -#ifndef fileOperations_hostCollatedFileOperation_H -#define fileOperations_hostCollatedFileOperation_H +#ifndef Foam_fileOperations_hostCollatedFileOperation_H +#define Foam_fileOperations_hostCollatedFileOperation_H #include "collatedFileOperation.H" @@ -80,7 +80,7 @@ class hostCollatedFileOperation // Private Data //- Communicator allocated/managed by us - label managedComm_; + mutable label managedComm_; // Private Member Functions @@ -100,11 +100,18 @@ public: // Constructors //- Default construct - explicit hostCollatedFileOperation(const bool verbose); + explicit hostCollatedFileOperation(bool verbose = false); //- Destructor virtual ~hostCollatedFileOperation(); + + + // Member Functions + + //- Transfer ownership of communicator to this fileOperation. + //- Use with caution + virtual void storeComm() const; }; @@ -112,6 +119,7 @@ public: Class hostCollatedFileOperationInitialise Declaration \*---------------------------------------------------------------------------*/ +//! Internal class only class hostCollatedFileOperationInitialise : public collatedFileOperationInitialise diff --git a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C index a69acf2514..25b35e3922 100644 --- a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C +++ b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2017-2018 OpenFOAM Foundation - Copyright (C) 2019-2022 OpenCFD Ltd. + Copyright (C) 2019-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -316,7 +316,7 @@ bool Foam::fileOperation::uniformFile(const fileNameList& names) bool Foam::fileOperation::uniformFile(const label comm, const fileName& name) { - if (!Pstream::parRun()) + if (!UPstream::parRun()) { return true; } @@ -1494,4 +1494,22 @@ Foam::label Foam::fileOperation::detectProcessorPath(const fileName& fName) } +// * * * * * * * * * * * * * * Friend Operators * * * * * * * * * * * * * * // + +Foam::Ostream& Foam::operator<< +( + Ostream& os, + const InfoProxy& iproxy +) +{ + const auto& fp = *iproxy; + + os << "fileHandler:" << fp.type() + // << " nProcs:" << fp.nProcs() + << " comm:" << fp.comm() + << " distributed:" << fp.distributed() << nl; + + return os; +} + // ************************************************************************* // diff --git a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.H b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.H index a62ad6644c..21b8d0e843 100644 --- a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.H +++ b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.H @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2017 OpenFOAM Foundation - Copyright (C) 2020-2022 OpenCFD Ltd. + Copyright (C) 2020-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -43,12 +43,14 @@ Description #include "ISstream.H" #include "Ostream.H" +#include "UPstream.H" #include "fileMonitor.H" #include "fileNameList.H" #include "instantList.H" #include "refPtr.H" #include "Enum.H" #include "Tuple2.H" +#include "InfoProxy.H" // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // @@ -56,11 +58,14 @@ namespace Foam { // Forward Declarations -class IOobject; -class regIOobject; +class fileOperation; class objectRegistry; +class regIOobject; +class IOobject; class Time; +Ostream& operator<<(Ostream& os, const InfoProxy& info); + /*---------------------------------------------------------------------------*\ Class fileOperation Declaration \*---------------------------------------------------------------------------*/ @@ -160,6 +165,22 @@ protected: //- Is either a directory (empty name()) or a file bool exists(IOobject& io) const; + //- Construction helper: check for locally allocated communicator + static inline label getManagedComm(const label communicator) + { + return + ( + ( + communicator < 0 + || communicator == UPstream::globalComm + || communicator == UPstream::selfComm + || communicator == UPstream::worldComm + ) + ? -1 + : communicator + ); + } + public: @@ -329,9 +350,18 @@ public: return old; } + //- Return info proxy, + //- used to print information to a stream + InfoProxy info() const noexcept { return *this; } + // Member Functions + //- Transfer ownership of communicator to this fileOperation. + //- Use with caution + virtual void storeComm() const = 0; + + // OSSpecific equivalents //- Make directory diff --git a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C index de391d0b0c..d73b453c96 100644 --- a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C +++ b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C @@ -740,10 +740,10 @@ masterUncollatedFileOperation UPstream::allocateCommunicator ( UPstream::worldComm, - subRanks(Pstream::nProcs()) + subRanks(UPstream::nProcs()) ) ), - managedComm_(comm_) + managedComm_(getManagedComm(comm_)) // Possibly locally allocated { init(verbose); } @@ -763,15 +763,19 @@ masterUncollatedFileOperation } +void Foam::fileOperations::masterUncollatedFileOperation::storeComm() const +{ + // From externally -> locally managed + managedComm_ = getManagedComm(comm_); +} + + // * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * // Foam::fileOperations::masterUncollatedFileOperation:: ~masterUncollatedFileOperation() { - if (UPstream::isUserComm(managedComm_)) - { - UPstream::freeCommunicator(managedComm_); - } + UPstream::freeCommunicator(managedComm_); } diff --git a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.H b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.H index 225102ab4d..0b4848488e 100644 --- a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.H +++ b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.H @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2017 OpenFOAM Foundation - Copyright (C) 2019-2022 OpenCFD Ltd. + Copyright (C) 2019-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -91,7 +91,7 @@ class masterUncollatedFileOperation // Private Data //- Communicator allocated/managed by us - label managedComm_; + mutable label managedComm_; // Private Member Functions @@ -99,6 +99,7 @@ class masterUncollatedFileOperation //- Any initialisation steps after constructing void init(bool verbose); + protected: // Protected Data @@ -491,7 +492,7 @@ public: // Constructors //- Default construct - explicit masterUncollatedFileOperation(bool verbose); + explicit masterUncollatedFileOperation(bool verbose = false); //- Construct from communicator masterUncollatedFileOperation(const label comm, bool verbose); @@ -503,6 +504,11 @@ public: // Member Functions + //- Transfer ownership of communicator to this fileOperation. + //- Use with caution + virtual void storeComm() const; + + // OSSpecific equivalents //- Make directory diff --git a/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C b/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C index 8acc5692ad..0988f3b251 100644 --- a/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C +++ b/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2017 OpenFOAM Foundation - Copyright (C) 2020-2022 OpenCFD Ltd. + Copyright (C) 2020-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -194,20 +194,24 @@ Foam::fileOperations::uncollatedFileOperation::uncollatedFileOperation ) : fileOperation(UPstream::worldComm), - managedComm_(-1) // worldComm is externally managed + managedComm_(-1) // Externally managed { init(verbose); } +void Foam::fileOperations::uncollatedFileOperation::storeComm() const +{ + // From externally -> locally managed + managedComm_ = getManagedComm(comm_); +} + + // * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * // Foam::fileOperations::uncollatedFileOperation::~uncollatedFileOperation() { - if (UPstream::isUserComm(managedComm_)) - { - UPstream::freeCommunicator(managedComm_); - } + UPstream::freeCommunicator(managedComm_); } diff --git a/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.H b/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.H index d81c54d86d..a58d60a58d 100644 --- a/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.H +++ b/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.H @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2017 OpenFOAM Foundation - Copyright (C) 2020-2022 OpenCFD Ltd. + Copyright (C) 2020-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -56,7 +56,7 @@ class uncollatedFileOperation // Private Data //- Communicator allocated/managed by us - label managedComm_; + mutable label managedComm_; // Private Member Functions @@ -97,7 +97,7 @@ public: // Constructors //- Default construct - explicit uncollatedFileOperation(bool verbose); + explicit uncollatedFileOperation(bool verbose = false); //- Destructor @@ -106,6 +106,11 @@ public: // Member Functions + //- Transfer ownership of communicator to this fileOperation. + //- Use with caution + virtual void storeComm() const; + + // OSSpecific equivalents //- Make directory