From 478c1b23125ffc5ec43c679a7ed5143d8389fb07 Mon Sep 17 00:00:00 2001 From: Mark Olesen Date: Wed, 23 Nov 2022 21:01:51 +0100 Subject: [PATCH] ENH: add atomic file creation support into masterOFstream (#2631) --- .../db/IOstreams/Fstreams/masterOFstream.C | 3 ++ .../db/IOstreams/Fstreams/masterOFstream.H | 31 +++++++++++++++++-- .../collatedFileOperation/OFstreamCollator.C | 8 ++++- .../collatedFileOperation/OFstreamCollator.H | 5 +++ .../collatedFileOperation.C | 3 ++ .../threadedCollatedOFstream.C | 22 +++++++++++++ .../threadedCollatedOFstream.H | 18 ++++++++++- .../fileOperation/fileOperation.H | 9 ++++++ .../masterUncollatedFileOperation.C | 23 ++++++++++++++ .../masterUncollatedFileOperation.H | 9 ++++++ .../uncollatedFileOperation.C | 13 ++++++++ .../uncollatedFileOperation.H | 11 ++++++- 12 files changed, 149 insertions(+), 6 deletions(-) diff --git a/src/OpenFOAM/db/IOstreams/Fstreams/masterOFstream.C b/src/OpenFOAM/db/IOstreams/Fstreams/masterOFstream.C index 88e6657b0d..eb7032a593 100644 --- a/src/OpenFOAM/db/IOstreams/Fstreams/masterOFstream.C +++ b/src/OpenFOAM/db/IOstreams/Fstreams/masterOFstream.C @@ -52,6 +52,7 @@ void Foam::masterOFstream::checkWrite OFstream os ( + atomic_, fName, IOstreamOption(IOstreamOption::BINARY, version(), compression_), append_ @@ -177,6 +178,7 @@ void Foam::masterOFstream::commit() Foam::masterOFstream::masterOFstream ( + IOstreamOption::atomicType atomic, const fileName& pathName, IOstreamOption streamOpt, IOstreamOption::appendType append, @@ -185,6 +187,7 @@ Foam::masterOFstream::masterOFstream : OStringStream(streamOpt), pathName_(pathName), + atomic_(atomic), compression_(streamOpt.compression()), append_(append), valid_(valid) diff --git a/src/OpenFOAM/db/IOstreams/Fstreams/masterOFstream.H b/src/OpenFOAM/db/IOstreams/Fstreams/masterOFstream.H index 7971ddea91..aa75227b94 100644 --- a/src/OpenFOAM/db/IOstreams/Fstreams/masterOFstream.H +++ b/src/OpenFOAM/db/IOstreams/Fstreams/masterOFstream.H @@ -58,13 +58,16 @@ class masterOFstream //- The backend file name const fileName pathName_; + //- Atomic file creation (ignored with append) + const IOstreamOption::atomicType atomic_; + //- Output file compression const IOstreamOption::compressionType compression_; //- Open file in append mode const IOstreamOption::appendType append_; - //- Should file be written + //- Should file be written (on this processor) const bool valid_; @@ -89,14 +92,36 @@ public: // Constructors - //- Construct from pathname and set stream status + //- Construct with specified atomic behaviour (with worldComm) + //- from pathname, stream option, optional append + masterOFstream + ( + IOstreamOption::atomicType atomic, + const fileName& pathname, + IOstreamOption streamOpt = IOstreamOption(), + IOstreamOption::appendType append = IOstreamOption::NON_APPEND, + const bool valid = true + ); + + //- Construct (with worldComm) + //- from pathname, stream option, optional append explicit masterOFstream ( const fileName& pathname, IOstreamOption streamOpt = IOstreamOption(), IOstreamOption::appendType append = IOstreamOption::NON_APPEND, const bool valid = true - ); + ) + : + masterOFstream + ( + IOstreamOption::NON_ATOMIC, + pathname, + streamOpt, + append, + valid + ) + {} //- Destructor - commits buffered information to file diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.C b/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.C index 8514aeb1e8..0ba8849772 100644 --- a/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.C +++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.C @@ -51,6 +51,7 @@ bool Foam::OFstreamCollator::writeFile const labelUList& recvSizes, const PtrList>& slaveData, // optional slave data IOstreamOption streamOpt, + IOstreamOption::atomicType atomic, IOstreamOption::appendType append, const dictionary& headerEntries ) @@ -76,7 +77,7 @@ bool Foam::OFstreamCollator::writeFile if (UPstream::master(comm)) { Foam::mkDir(fName.path()); - osPtr.reset(new OFstream(fName, streamOpt, append)); + osPtr.reset(new OFstream(atomic, fName, streamOpt, append)); auto& os = *osPtr; if (append == IOstreamOption::NON_APPEND) @@ -213,6 +214,7 @@ void* Foam::OFstreamCollator::writeAll(void *threadarg) ptr->sizes_, slaveData, ptr->streamOpt_, + ptr->atomic_, ptr->append_, ptr->headerEntries_ ); @@ -348,6 +350,7 @@ bool Foam::OFstreamCollator::write const fileName& fName, const string& data, IOstreamOption streamOpt, + IOstreamOption::atomicType atomic, IOstreamOption::appendType append, const bool useThread, const dictionary& headerEntries @@ -387,6 +390,7 @@ bool Foam::OFstreamCollator::write recvSizes, dummySlaveData, streamOpt, + atomic, append, headerEntries ); @@ -425,6 +429,7 @@ bool Foam::OFstreamCollator::write ), recvSizes, streamOpt, + atomic, append, headerEntries ) @@ -550,6 +555,7 @@ bool Foam::OFstreamCollator::write data, recvSizes, streamOpt, + atomic, append, headerEntries ) diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.H b/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.H index f131cdec28..923fb4c7c1 100644 --- a/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.H +++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.H @@ -81,6 +81,7 @@ class OFstreamCollator const labelList sizes_; PtrList> slaveData_; const IOstreamOption streamOpt_; + IOstreamOption::atomicType atomic_; IOstreamOption::appendType append_; const dictionary headerEntries_; @@ -92,6 +93,7 @@ class OFstreamCollator const string& data, const labelList& sizes, IOstreamOption streamOpt, + IOstreamOption::atomicType atomic, IOstreamOption::appendType append, const dictionary& headerEntries ) @@ -103,6 +105,7 @@ class OFstreamCollator sizes_(sizes), slaveData_(), streamOpt_(streamOpt), + atomic_(atomic), append_(append), headerEntries_(headerEntries) {} @@ -157,6 +160,7 @@ class OFstreamCollator const labelUList& recvSizes, const PtrList>& slaveData, IOstreamOption streamOpt, + IOstreamOption::atomicType atomic, IOstreamOption::appendType append, const dictionary& headerEntries ); @@ -200,6 +204,7 @@ public: const fileName&, const string& data, IOstreamOption streamOpt, + IOstreamOption::atomicType atomic, IOstreamOption::appendType append, const bool useThread = true, const dictionary& headerEntries = dictionary::null diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C b/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C index edfbec5563..834bab9333 100644 --- a/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C +++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C @@ -381,6 +381,7 @@ bool Foam::fileOperations::collatedFileOperation::writeObject << endl; } + // Note: currently still NON_ATOMIC (Dec-2022) masterOFstream os ( pathName, @@ -424,6 +425,7 @@ bool Foam::fileOperations::collatedFileOperation::writeObject << endl; } + // Note: currently still NON_ATOMIC (Dec-2022) masterOFstream os ( pathName, @@ -481,6 +483,7 @@ bool Foam::fileOperations::collatedFileOperation::writeObject writer_.waitAll(); } + // Note: currently still NON_ATOMIC (Dec-2022) threadedCollatedOFstream os ( writer_, diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/threadedCollatedOFstream.C b/src/OpenFOAM/global/fileOperations/collatedFileOperation/threadedCollatedOFstream.C index bf8e046e74..57af56c268 100644 --- a/src/OpenFOAM/global/fileOperations/collatedFileOperation/threadedCollatedOFstream.C +++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/threadedCollatedOFstream.C @@ -35,6 +35,7 @@ License Foam::threadedCollatedOFstream::threadedCollatedOFstream ( OFstreamCollator& writer, + IOstreamOption::atomicType atomic, const fileName& pathName, IOstreamOption streamOpt, const bool useThread @@ -43,12 +44,32 @@ Foam::threadedCollatedOFstream::threadedCollatedOFstream OStringStream(streamOpt), writer_(writer), pathName_(pathName), + atomic_(atomic), compression_(streamOpt.compression()), useThread_(useThread), headerEntries_() {} +Foam::threadedCollatedOFstream::threadedCollatedOFstream +( + OFstreamCollator& writer, + const fileName& pathName, + IOstreamOption streamOpt, + const bool useThread +) +: + threadedCollatedOFstream + ( + writer, + IOstreamOption::NON_ATOMIC, + pathName, + streamOpt, + useThread + ) +{} + + // * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * // Foam::threadedCollatedOFstream::~threadedCollatedOFstream() @@ -59,6 +80,7 @@ Foam::threadedCollatedOFstream::~threadedCollatedOFstream() pathName_, str(), IOstreamOption(IOstreamOption::BINARY, version(), compression_), + atomic_, IOstreamOption::NON_APPEND, useThread_, headerEntries_ diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/threadedCollatedOFstream.H b/src/OpenFOAM/global/fileOperations/collatedFileOperation/threadedCollatedOFstream.H index df703371a2..ad91139b83 100644 --- a/src/OpenFOAM/global/fileOperations/collatedFileOperation/threadedCollatedOFstream.H +++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/threadedCollatedOFstream.H @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2017-2018 OpenFOAM Foundation - Copyright (C) 2021 OpenCFD Ltd. + Copyright (C) 2021-2022 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -62,10 +62,16 @@ class threadedCollatedOFstream //- The backend writer OFstreamCollator& writer_; + //- The backend file name const fileName pathName_; + //- Atomic file creation (ignored with append) + const IOstreamOption::atomicType atomic_; + + //- Output file compression const IOstreamOption::compressionType compression_; + //- Use threading const bool useThread_; //- Additional FoamFile entries for decomposed data @@ -76,6 +82,16 @@ public: // Constructors + //- Construct and set stream status + threadedCollatedOFstream + ( + OFstreamCollator& writer, + IOstreamOption::atomicType atomic, + const fileName& pathname, + IOstreamOption streamOpt = IOstreamOption(), + const bool useThread = true + ); + //- Construct and set stream status threadedCollatedOFstream ( diff --git a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.H b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.H index d763c2e9db..9cfebf5ee3 100644 --- a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.H +++ b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.H @@ -471,6 +471,15 @@ public: const bool valid = true ) const = 0; + //- Generate an OSstream that writes a file + virtual autoPtr NewOFstream + ( + IOstreamOption::atomicType atomic, + const fileName& pathname, + IOstreamOption streamOpt = IOstreamOption(), + const bool valid = true + ) const = 0; + // File modification checking diff --git a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C index 5205de0043..7be6376da4 100644 --- a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C +++ b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C @@ -2462,6 +2462,29 @@ Foam::fileOperations::masterUncollatedFileOperation::NewOFstream } +Foam::autoPtr +Foam::fileOperations::masterUncollatedFileOperation::NewOFstream +( + IOstreamOption::atomicType atomic, + const fileName& pathName, + IOstreamOption streamOpt, + const bool valid +) const +{ + return autoPtr + ( + new masterOFstream + ( + atomic, + pathName, + streamOpt, + IOstreamOption::NON_APPEND, + valid + ) + ); +} + + void Foam::fileOperations::masterUncollatedFileOperation::flush() const { fileOperation::flush(); diff --git a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.H b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.H index 844bb26c0f..fc1ca689fd 100644 --- a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.H +++ b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.H @@ -700,6 +700,15 @@ public: const bool valid = true ) const; + //- Generate an OSstream that writes a file + virtual autoPtr NewOFstream + ( + IOstreamOption::atomicType atomic, + const fileName& pathname, + IOstreamOption streamOpt = IOstreamOption(), + const bool valid = true + ) const; + // File modification checking diff --git a/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C b/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C index 0b0519deca..41777a6188 100644 --- a/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C +++ b/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C @@ -705,4 +705,17 @@ Foam::fileOperations::uncollatedFileOperation::NewOFstream } +Foam::autoPtr +Foam::fileOperations::uncollatedFileOperation::NewOFstream +( + IOstreamOption::atomicType atomic, + const fileName& pathName, + IOstreamOption streamOpt, + const bool valid +) const +{ + return autoPtr(new OFstream(atomic, pathName, streamOpt)); +} + + // ************************************************************************* // diff --git a/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.H b/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.H index f2efa472f0..442a446367 100644 --- a/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.H +++ b/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.H @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2017 OpenFOAM Foundation - Copyright (C) 2020-2021 OpenCFD Ltd. + Copyright (C) 2020-2022 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -290,6 +290,15 @@ public: IOstreamOption streamOpt = IOstreamOption(), const bool valid = true ) const; + + //- Generate an OSstream that writes a file + virtual autoPtr NewOFstream + ( + IOstreamOption::atomicType, + const fileName& pathname, + IOstreamOption streamOpt = IOstreamOption(), + const bool valid = true + ) const; };