From 5338e56c7361295ce2a341f672bc8734d856e873 Mon Sep 17 00:00:00 2001 From: Mark Olesen Date: Tue, 8 Nov 2022 21:01:22 +0100 Subject: [PATCH] ENH: add support for OFstream atomic file creation (#2631) - with ATOMIC, an intermediary file is created - eg, (fileAbc~tmp~) where all of the output is written to. When the stream goes out of scope, this intermediary file is moved/renamed to the actually output name - eg, (fileAbc~tmp~) -> (fileAbc). This adds some safety if the simulation crashes while writing the file, since it will the partial (corrupt) file will be left behind as (fileAbc~tmp~) and not as (fileAbc), which means it will will be treated as a backup file and not loaded again on restart. ENH: provided enumeration for APPEND/NON_APPEND - clearer than using bool (with comments). Since append mode is primarily only used by masterOFstream etc this change is unlikely to affect user coding. ENH: use file atomic for ensight file creation - avoids corrupt (truncated) files being referenced by the ensight case file if the simulation crashes while writing the ensight file. --- applications/test/OFstream/Make/files | 3 + applications/test/OFstream/Make/options | 2 + applications/test/OFstream/Test-OFstream.C | 228 ++++++++++++++++ src/OpenFOAM/db/IOstreams/Fstreams/IFstream.C | 4 +- src/OpenFOAM/db/IOstreams/Fstreams/IFstream.H | 4 +- src/OpenFOAM/db/IOstreams/Fstreams/OFstream.C | 26 +- src/OpenFOAM/db/IOstreams/Fstreams/OFstream.H | 44 +++- .../db/IOstreams/Fstreams/fstreamPointer.H | 47 ++-- .../db/IOstreams/Fstreams/fstreamPointers.C | 249 +++++++++++++----- .../db/IOstreams/Fstreams/masterOFstream.C | 6 +- .../db/IOstreams/Fstreams/masterOFstream.H | 7 +- .../db/IOstreams/IOstreams/IOstream.C | 2 +- .../db/IOstreams/IOstreams/IOstreamOption.C | 2 +- .../db/IOstreams/IOstreams/IOstreamOption.H | 19 +- .../collatedFileOperation/OFstreamCollator.C | 8 +- .../collatedFileOperation/OFstreamCollator.H | 14 +- .../collatedFileOperation.C | 12 +- .../threadedCollatedOFstream.C | 2 +- .../masterUncollatedFileOperation.C | 2 +- src/fileFormats/ensight/file/ensightFile.C | 4 +- src/fileFormats/ensight/file/ensightFile.H | 4 +- src/fileFormats/ensight/file/ensightGeoFile.H | 2 +- .../movement/lumpedPointMovement.C | 2 +- 23 files changed, 559 insertions(+), 134 deletions(-) create mode 100644 applications/test/OFstream/Make/files create mode 100644 applications/test/OFstream/Make/options create mode 100644 applications/test/OFstream/Test-OFstream.C diff --git a/applications/test/OFstream/Make/files b/applications/test/OFstream/Make/files new file mode 100644 index 0000000000..7c5032c730 --- /dev/null +++ b/applications/test/OFstream/Make/files @@ -0,0 +1,3 @@ +Test-OFstream.C + +EXE = $(FOAM_USER_APPBIN)/Test-OFstream diff --git a/applications/test/OFstream/Make/options b/applications/test/OFstream/Make/options new file mode 100644 index 0000000000..18e6fe47af --- /dev/null +++ b/applications/test/OFstream/Make/options @@ -0,0 +1,2 @@ +/* EXE_INC = */ +/* EXE_LIBS = */ diff --git a/applications/test/OFstream/Test-OFstream.C b/applications/test/OFstream/Test-OFstream.C new file mode 100644 index 0000000000..ac108699b0 --- /dev/null +++ b/applications/test/OFstream/Test-OFstream.C @@ -0,0 +1,228 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | OpenFOAM: The Open Source CFD Toolbox + \\ / O peration | + \\ / A nd | www.openfoam.com + \\/ M anipulation | +------------------------------------------------------------------------------- + Copyright (C) 2022 OpenCFD Ltd. +------------------------------------------------------------------------------- +License + This file is part of OpenFOAM. + + OpenFOAM is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with OpenFOAM. If not, see . + +Description + Test OFstream. Primarily atomic operations + +\*---------------------------------------------------------------------------*/ + +#include "Fstream.H" +#include "IOstreams.H" +#include "OSspecific.H" +#include "argList.H" +#include "ListOps.H" + +using namespace Foam; + +void listFiles(const fileName& dir) +{ + wordList files = ListOps::create + ( + readDir(dir, fileName::FILE), + nameOp() + ); + + Info + << nl + << "files:" << nl + << files << nl + << "ls" << nl + << "============" << endl; + Foam::system("ls -al " + dir); + Info<< "============" << endl; +} + + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // +// Main program: + +int main(int argc, char *argv[]) +{ + argList::addBoolOption("gz", "Use compression"); + argList::addBoolOption("append", "Use append mode"); + argList::addBoolOption("atomic", "Use atomic"); + argList::addBoolOption("keep", "Do not remove test directory"); + argList::addOption("write", "file", "test writing to file"); + + #include "setRootCase.H" + + const fileName baseDir("Test-OFstream-directory"); + + Foam::mkDir(baseDir); + + InfoErr<< "mkdir: " << baseDir << endl; + + IOstreamOption streamOpt; + + if (args.found("gz")) + { + streamOpt.compression(IOstreamOption::COMPRESSED); + } + + IOstreamOption::appendType append = + ( + args.found("append") + ? IOstreamOption::APPEND + : IOstreamOption::NON_APPEND + ); + IOstreamOption::atomicType atomic = + ( + args.found("atomic") + ? IOstreamOption::ATOMIC + : IOstreamOption::NON_ATOMIC + ); + + { + OFstream(baseDir/"dummy")() << "Some file content" << endl; + + Foam::ln("dummy", baseDir/"Test2.txt"); + Foam::ln("dummy", baseDir/"Test3.txt"); + Foam::ln("dummy", baseDir/"Test4.txt"); + Foam::ln("dummy", baseDir/"Test4.txt.gz"); + Foam::ln("dummy", baseDir/"Test5.txt"); + Foam::ln("dummy", baseDir/"Test5.txt.gz"); + } + + { + OFstream os + ( + atomic, + baseDir/"Test1.txt", + streamOpt, + append + ); + + os << "=========================" << endl; + + InfoErr<< "open: " << os.name() << endl; + InfoErr<< "... sleep" << endl; + + listFiles(baseDir); + + sleep(2); + + os << "+++++++++++++++++++++++++++++++++++" << endl; + } + + { + OFstream os + ( + atomic, + baseDir/"Test2.txt", + streamOpt + // NON_APPEND + ); + + os << "=========================" << endl; + + InfoErr<< "open: " << os.name() << endl; + InfoErr<< "... sleep" << endl; + + listFiles(baseDir); + + sleep(2); + + os << "+++++++++++++++++++++++++++++++++++" << endl; + } + { + OFstream os + ( + atomic, + baseDir/"Test3.txt", + streamOpt, + IOstreamOption::APPEND + ); + + os << "=========================" << endl; + + InfoErr<< "open: " << os.name() << endl; + InfoErr<< "... sleep" << endl; + + listFiles(baseDir); + + sleep(2); + + os << "+++++++++++++++++++++++++++++++++++" << endl; + } + { + OFstream os + ( + baseDir/"Test4.txt", + IOstreamOption::ASCII, + IOstreamOption::COMPRESSED + ); + + os << "=========================" << endl; + + InfoErr<< "open: " << os.name() << endl; + InfoErr<< "... sleep" << endl; + + listFiles(baseDir); + + sleep(2); + + os << "+++++++++++++++++++++++++++++++++++" << endl; + } + { + OFstream os + ( + IOstreamOption::ATOMIC, + baseDir/"Test5.txt" + // ASCII UNCOMPRESSED NON_APPEND + ); + + os << "=========================" << endl; + + InfoErr<< "open: " << os.name() << endl; + InfoErr<< "... sleep" << endl; + + listFiles(baseDir); + + sleep(2); + + os << "+++++++++++++++++++++++++++++++++++" << endl; + } + + Info<< nl << "done:" << endl; + + listFiles(baseDir); + + if (args.found("keep")) + { + InfoErr<< "keep: " << baseDir << endl; + } + else + { + InfoErr<< "rmdir: " << baseDir << endl; + Foam::rmDir(baseDir); + } + + Info<< "\nEnd\n" << endl; + + return 0; +} + + +// ************************************************************************* // diff --git a/src/OpenFOAM/db/IOstreams/Fstreams/IFstream.C b/src/OpenFOAM/db/IOstreams/Fstreams/IFstream.C index 8829cbc877..e59bad6f8f 100644 --- a/src/OpenFOAM/db/IOstreams/Fstreams/IFstream.C +++ b/src/OpenFOAM/db/IOstreams/Fstreams/IFstream.C @@ -126,7 +126,7 @@ void Foam::IFstream::rewind() if (IOstreamOption::COMPRESSED == ifstreamPointer::whichCompression()) { lineNumber_ = 1; // Reset line number - ifstreamPointer::reopen_gz(this->name() + ".gz"); + ifstreamPointer::reopen_gz(this->name()); setState(ifstreamPointer::get()->rdstate()); } else @@ -150,7 +150,7 @@ Foam::IFstream& Foam::IFstream::operator()() const if (!good()) { // Also checks .gz file - if (isFile(this->name(), true)) + if (Foam::isFile(this->name(), true)) { check(FUNCTION_NAME); FatalIOError.exit(); diff --git a/src/OpenFOAM/db/IOstreams/Fstreams/IFstream.H b/src/OpenFOAM/db/IOstreams/Fstreams/IFstream.H index 989ebaf77e..f178fd95cc 100644 --- a/src/OpenFOAM/db/IOstreams/Fstreams/IFstream.H +++ b/src/OpenFOAM/db/IOstreams/Fstreams/IFstream.H @@ -64,14 +64,14 @@ public: // Constructors - //- Construct from pathname + //- Construct from pathname, default or specified stream options explicit IFstream ( const fileName& pathname, IOstreamOption streamOpt = IOstreamOption() ); - //- Construct from pathname, format (version) + //- Construct from pathname and format IFstream ( const fileName& pathname, diff --git a/src/OpenFOAM/db/IOstreams/Fstreams/OFstream.C b/src/OpenFOAM/db/IOstreams/Fstreams/OFstream.C index 880651ba8f..fc65cfd462 100644 --- a/src/OpenFOAM/db/IOstreams/Fstreams/OFstream.C +++ b/src/OpenFOAM/db/IOstreams/Fstreams/OFstream.C @@ -56,12 +56,19 @@ Foam::OFstream::OFstream Foam::OFstream::OFstream ( + IOstreamOption::atomicType atomic, const fileName& pathname, IOstreamOption streamOpt, - const bool append + IOstreamOption::appendType append ) : - Foam::ofstreamPointer(pathname, streamOpt.compression(), append), + Foam::ofstreamPointer + ( + pathname, + streamOpt.compression(), + (IOstreamOption::appendType::APPEND == append), + (IOstreamOption::atomicType::ATOMIC == atomic) + ), OSstream(*(ofstreamPointer::get()), pathname, streamOpt) { setClosed(); @@ -100,7 +107,9 @@ Foam::OFstream::OFstream // * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * // Foam::OFstream::~OFstream() -{} +{ + ofstreamPointer::close(this->name()); +} // * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // @@ -137,15 +146,8 @@ const std::ostream& Foam::OFstream::stdStream() const void Foam::OFstream::rewind() { - if (IOstreamOption::COMPRESSED == ofstreamPointer::whichCompression()) - { - ofstreamPointer::reopen_gz(this->name() + ".gz"); - } - else - { - // Reopen (truncate) - ofstreamPointer::reopen(this->name()); - } + // Reopen (truncate) std::ostream + ofstreamPointer::reopen(this->name()); // As per OSstream::rewind() diff --git a/src/OpenFOAM/db/IOstreams/Fstreams/OFstream.H b/src/OpenFOAM/db/IOstreams/Fstreams/OFstream.H index 674fc97cd5..488c878471 100644 --- a/src/OpenFOAM/db/IOstreams/Fstreams/OFstream.H +++ b/src/OpenFOAM/db/IOstreams/Fstreams/OFstream.H @@ -68,28 +68,57 @@ public: // Behaves like \c /dev/null and is named accordingly explicit OFstream(std::nullptr_t); - //- Construct from pathname + //- Construct with specified atomic behaviour + //- from pathname, stream option, optional append + OFstream + ( + IOstreamOption::atomicType atomic, + const fileName& pathname, + IOstreamOption streamOpt = IOstreamOption(), + IOstreamOption::appendType append = IOstreamOption::NON_APPEND + ); + + //- Construct from pathname and other specifications explicit OFstream ( const fileName& pathname, IOstreamOption streamOpt = IOstreamOption(), - const bool append = false - ); + IOstreamOption::appendType append = IOstreamOption::NON_APPEND + ) + : + OFstream(IOstreamOption::NON_ATOMIC, pathname, streamOpt, append) + {} - //- Construct from pathname, format (uncompressed), optional append + //- Construct from pathname, format (uncompressed), optional append, + //- atomic behaviour as per system default OFstream ( const fileName& pathname, IOstreamOption::streamFormat fmt, IOstreamOption::compressionType cmp = IOstreamOption::UNCOMPRESSED, - const bool append = false + IOstreamOption::appendType append = IOstreamOption::NON_APPEND ) : OFstream(pathname, IOstreamOption(fmt, cmp), append) {} + //- Construct with specified atomic behaviour + //- from pathname, format (uncompressed), optional append + OFstream + ( + IOstreamOption::atomicType atomic, + const fileName& pathname, + IOstreamOption::streamFormat fmt, + IOstreamOption::compressionType cmp = IOstreamOption::UNCOMPRESSED, + IOstreamOption::appendType append = IOstreamOption::NON_APPEND + ) + : + OFstream(atomic, pathname, IOstreamOption(fmt, cmp), append) + {} - //- Destructor + + //- Destructor. Possibly invokes an atomic rename + //- (preference defined during construction) ~OFstream(); @@ -122,13 +151,14 @@ public: #ifdef Foam_IOstream_extras //- Construct from pathname, format (version, compression) + FOAM_DEPRECATED_FOR(2022-09, "Construct without specifying version") OFstream ( const fileName& pathname, IOstreamOption::streamFormat fmt, IOstreamOption::versionNumber ver, IOstreamOption::compressionType cmp = IOstreamOption::UNCOMPRESSED, - const bool append = false + IOstreamOption::appendType append = IOstreamOption::NON_APPEND ) : OFstream(pathname, IOstreamOption(fmt, ver, cmp), append) diff --git a/src/OpenFOAM/db/IOstreams/Fstreams/fstreamPointer.H b/src/OpenFOAM/db/IOstreams/Fstreams/fstreamPointer.H index 6fb70c0b0a..4fb72dbd3e 100644 --- a/src/OpenFOAM/db/IOstreams/Fstreams/fstreamPointer.H +++ b/src/OpenFOAM/db/IOstreams/Fstreams/fstreamPointer.H @@ -5,7 +5,7 @@ \\ / A nd | www.openfoam.com \\/ M anipulation | ------------------------------------------------------------------------------- - Copyright (C) 2020 OpenCFD Ltd. + Copyright (C) 2020-2022 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -77,12 +77,14 @@ class ifstreamPointer //- The stream pointer (ifstream or igzstream) std::unique_ptr ptr_; + protected: // Protected Member Functions //- Special 'rewind' method for compressed stream - void reopen_gz(const std::string& pathname_gz); + void reopen_gz(const std::string& pathname); + public: @@ -109,7 +111,9 @@ public: // Constructors - //- Construct from pathname + //- Construct from pathname. + // Attempts to read the specified file. + // If that fails, try as a compressed file (.gz ending). explicit ifstreamPointer(const fileName& pathname); @@ -167,23 +171,25 @@ class ofstreamPointer //- The stream pointer (ofstream | ogzstream | ocountstream) std::unique_ptr ptr_; + //- Atomic file creation + bool atomic_; + + protected: // Protected Member Functions - //- Special 'rewind' method for compressed stream - void reopen_gz(const std::string& pathname_gz); - - //- General 'rewind' method (non-compressed) + //- Reopen for compressed/non-compressed void reopen(const std::string& pathname); + //- Close stream and rename file + void close(const std::string& pathname); + + public: // Generated Methods - //- Default construct (empty) - ofstreamPointer() noexcept = default; - //- No copy construct ofstreamPointer(const ofstreamPointer&) = delete; @@ -202,22 +208,25 @@ public: // Constructors + //- Default construct (empty) + ofstreamPointer() noexcept; + //- Construct as null output stream using Foam::ocountstream explicit ofstreamPointer(std::nullptr_t); - //- Construct from pathname, with specified append option - ofstreamPointer(const fileName& pathname, const bool append) - : - ofstreamPointer(pathname, IOstreamOption::UNCOMPRESSED, append) - {} - - //- Construct from pathname, - //- with preferred compression and specified append option + //- Construct from pathname, compression, append, file handling atomic + // \param pathname The file name to open for writing + // \param comp UNCOMPRESSED | COMPRESSED + // \param append Open in append mode + // \param atomic Write into temporary file (not target file). + // This option should only be used with a stream wrapper + // (eg, OFstream) that handles the final renaming. explicit ofstreamPointer ( const fileName& pathname, IOstreamOption::compressionType comp = IOstreamOption::UNCOMPRESSED, - const bool append = false + const bool append = false, + const bool atomic = false ); diff --git a/src/OpenFOAM/db/IOstreams/Fstreams/fstreamPointers.C b/src/OpenFOAM/db/IOstreams/Fstreams/fstreamPointers.C index 612f97ca3a..16815f7a0a 100644 --- a/src/OpenFOAM/db/IOstreams/Fstreams/fstreamPointers.C +++ b/src/OpenFOAM/db/IOstreams/Fstreams/fstreamPointers.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2011 OpenFOAM Foundation - Copyright (C) 2018-2021 OpenCFD Ltd. + Copyright (C) 2018-2022 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -29,6 +29,7 @@ License #include "fstreamPointer.H" #include "OCountStream.H" #include "OSspecific.H" +#include // HAVE_LIBZ defined externally // #define HAVE_LIBZ @@ -37,37 +38,6 @@ License #include "gzstream.h" #endif /* HAVE_LIBZ */ -// * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * // - -namespace Foam -{ - static inline void removeConflictingFiles - ( - const fileName& otherName, - const bool append, - const fileName& targetName - ) - { - // Remove other (compressed/uncompressed) version - - const fileName::Type pathType = Foam::type(otherName, false); - - if (pathType == fileName::FILE || pathType == fileName::SYMLINK) - { - Foam::rm(otherName); - } - - // Disallow writing into symlinked files. - // Eg, avoid problems with symlinked initial fields - - if (!append && Foam::type(targetName, false) == fileName::SYMLINK) - { - Foam::rm(targetName); - } - } -} - - // * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * // bool Foam::ifstreamPointer::supports_gz() @@ -112,7 +82,7 @@ Foam::ifstreamPointer::ifstreamPointer const fileName pathname_gz(pathname + ".gz"); - if (isFile(pathname_gz, false)) + if (Foam::isFile(pathname_gz, false)) { #ifdef HAVE_LIBZ @@ -128,13 +98,29 @@ Foam::ifstreamPointer::ifstreamPointer #endif /* HAVE_LIBZ */ } + else + { + // TBD: + // Can also fallback and open .orig files too + // + // auto* file = dynamic_cast(ptr_.get()); + // file->open(pathname + ".orig", mode); + } } } +Foam::ofstreamPointer::ofstreamPointer() noexcept +: + ptr_(), + atomic_(false) +{} + + Foam::ofstreamPointer::ofstreamPointer(std::nullptr_t) : - ptr_(new Foam::ocountstream) + ptr_(new Foam::ocountstream), + atomic_(false) {} @@ -142,10 +128,12 @@ Foam::ofstreamPointer::ofstreamPointer ( const fileName& pathname, IOstreamOption::compressionType comp, - const bool append + const bool append, + const bool atomic ) : - ptr_(nullptr) + ptr_(nullptr), + atomic_(atomic) { std::ios_base::openmode mode ( @@ -155,18 +143,56 @@ Foam::ofstreamPointer::ofstreamPointer if (append) { mode |= std::ios_base::app; + + // Cannot append to gzstream + comp = IOstreamOption::UNCOMPRESSED; + + // Cannot use append + atomic operation, without lots of extra work + atomic_ = false; } + + // When opening new files, remove file variants out of the way. + // Eg, opening "file1" + // - remove old "file1.gz" (compressed) + // - also remove old "file1" if it is a symlink and we are not appending + // + // Not writing into symlinked files avoids problems with symlinked + // initial fields (eg, 0/U -> ../0.orig/U) + const fileName pathname_gz(pathname + ".gz"); + const fileName pathname_tmp(pathname + "~tmp~"); + + fileName::Type fType = fileName::Type::UNDEFINED; if (IOstreamOption::COMPRESSED == comp) { - // Output compression requested + // Output compression requested. #ifdef HAVE_LIBZ + // TBD: + // atomic_ = true; // Always treat COMPRESSED like an atomic - removeConflictingFiles(pathname, append, pathname_gz); - ptr_.reset(new ogzstream(pathname_gz, mode)); + const fileName& target = (atomic_ ? pathname_tmp : pathname_gz); + + // Remove old uncompressed version (if any) + fType = Foam::type(pathname, false); + if (fType == fileName::SYMLINK || fType == fileName::FILE) + { + Foam::rm(pathname); + } + + // Avoid writing into symlinked files (non-append mode) + if (!append || atomic_) + { + fType = Foam::type(target, false); + if (fType == fileName::SYMLINK) + { + Foam::rm(target); + } + } + + ptr_.reset(new ogzstream(target, mode)); #else /* HAVE_LIBZ */ @@ -181,43 +207,51 @@ Foam::ofstreamPointer::ofstreamPointer #endif /* HAVE_LIBZ */ } - if (IOstreamOption::UNCOMPRESSED == comp) + if (IOstreamOption::COMPRESSED != comp) { - removeConflictingFiles(pathname_gz, append, pathname); - ptr_.reset(new std::ofstream(pathname, mode)); + const fileName& target = (atomic_ ? pathname_tmp : pathname); + + // Remove old compressed version (if any) + fType = Foam::type(pathname_gz, false); + if (fType == fileName::SYMLINK || fType == fileName::FILE) + { + Foam::rm(pathname_gz); + } + + // Avoid writing into symlinked files (non-append mode) + if (!append || atomic_) + { + fType = Foam::type(target, false); + if (fType == fileName::SYMLINK) + { + Foam::rm(target); + } + } + + ptr_.reset(new std::ofstream(target, mode)); } } // * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // -void Foam::ifstreamPointer::reopen_gz(const std::string& pathname_gz) +void Foam::ifstreamPointer::reopen_gz(const std::string& pathname) { #ifdef HAVE_LIBZ - igzstream* gz = dynamic_cast(ptr_.get()); + auto* gz = dynamic_cast(ptr_.get()); if (gz) { // Special treatment for gzstream gz->close(); gz->clear(); - gz->open(pathname_gz); - } - #endif /* HAVE_LIBZ */ -} - -void Foam::ofstreamPointer::reopen_gz(const std::string& pathname_gz) -{ - #ifdef HAVE_LIBZ - ogzstream* gz = dynamic_cast(ptr_.get()); - - if (gz) - { - // Special treatment for gzstream - gz->close(); - gz->clear(); - gz->open(pathname_gz); + gz->open + ( + pathname + ".gz", + (std::ios_base::in | std::ios_base::binary) + ); + return; } #endif /* HAVE_LIBZ */ } @@ -225,7 +259,36 @@ void Foam::ofstreamPointer::reopen_gz(const std::string& pathname_gz) void Foam::ofstreamPointer::reopen(const std::string& pathname) { - std::ofstream* file = dynamic_cast(ptr_.get()); + #ifdef HAVE_LIBZ + auto* gz = dynamic_cast(ptr_.get()); + + if (gz) + { + // Special treatment for gzstream + gz->close(); + gz->clear(); + + if (atomic_) + { + gz->open + ( + pathname + "~tmp~", + (std::ios_base::out | std::ios_base::binary) + ); + } + else + { + gz->open + ( + pathname + ".gz", + (std::ios_base::out | std::ios_base::binary) + ); + } + return; + } + #endif /* HAVE_LIBZ */ + + auto* file = dynamic_cast(ptr_.get()); if (file) { @@ -234,7 +297,69 @@ void Foam::ofstreamPointer::reopen(const std::string& pathname) file->close(); } file->clear(); - file->open(pathname); + + // Don't need original request to append since rewind implies + // trashing that anyhow. + + if (atomic_) + { + file->open + ( + pathname + "~tmp~", + (std::ios_base::out | std::ios_base::binary) + ); + } + else + { + file->open + ( + pathname, + (std::ios_base::out | std::ios_base::binary) + ); + } + return; + } +} + + +void Foam::ofstreamPointer::close(const std::string& pathname) +{ + if (!atomic_ || pathname.empty()) return; + + #ifdef HAVE_LIBZ + auto* gz = dynamic_cast(ptr_.get()); + + if (gz) + { + // Special treatment for gzstream + gz->close(); + gz->clear(); + + std::rename + ( + (pathname + "~tmp~").c_str(), + (pathname + ".gz").c_str() + ); + return; + } + #endif /* HAVE_LIBZ */ + + auto* file = dynamic_cast(ptr_.get()); + + if (file) + { + if (file->is_open()) + { + file->close(); + } + file->clear(); + + std::rename + ( + (pathname + "~tmp~").c_str(), + pathname.c_str() + ); + return; } } diff --git a/src/OpenFOAM/db/IOstreams/Fstreams/masterOFstream.C b/src/OpenFOAM/db/IOstreams/Fstreams/masterOFstream.C index cc2d5a472a..88e6657b0d 100644 --- a/src/OpenFOAM/db/IOstreams/Fstreams/masterOFstream.C +++ b/src/OpenFOAM/db/IOstreams/Fstreams/masterOFstream.C @@ -83,7 +83,7 @@ void Foam::masterOFstream::checkWrite const std::string& s ) { - checkWrite(fName, &s[0], s.length()); + checkWrite(fName, s.data(), s.length()); } @@ -126,7 +126,7 @@ void Foam::masterOFstream::commit() string s(this->str()); this->reset(); - os.write(&s[0], s.length()); + os.write(s.data(), s.length()); } labelList recvSizes; @@ -179,7 +179,7 @@ Foam::masterOFstream::masterOFstream ( const fileName& pathName, IOstreamOption streamOpt, - const bool append, + IOstreamOption::appendType append, const bool valid ) : diff --git a/src/OpenFOAM/db/IOstreams/Fstreams/masterOFstream.H b/src/OpenFOAM/db/IOstreams/Fstreams/masterOFstream.H index 2c2f9c3667..7971ddea91 100644 --- a/src/OpenFOAM/db/IOstreams/Fstreams/masterOFstream.H +++ b/src/OpenFOAM/db/IOstreams/Fstreams/masterOFstream.H @@ -55,11 +55,14 @@ class masterOFstream { // Private Data + //- The backend file name const fileName pathName_; + //- Output file compression const IOstreamOption::compressionType compression_; - const bool append_; + //- Open file in append mode + const IOstreamOption::appendType append_; //- Should file be written const bool valid_; @@ -91,7 +94,7 @@ public: ( const fileName& pathname, IOstreamOption streamOpt = IOstreamOption(), - const bool append = false, + IOstreamOption::appendType append = IOstreamOption::NON_APPEND, const bool valid = true ); diff --git a/src/OpenFOAM/db/IOstreams/IOstreams/IOstream.C b/src/OpenFOAM/db/IOstreams/IOstreams/IOstream.C index c126ab1d99..983c3a7275 100644 --- a/src/OpenFOAM/db/IOstreams/IOstreams/IOstream.C +++ b/src/OpenFOAM/db/IOstreams/IOstreams/IOstream.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2011-2015 OpenFOAM Foundation - Copyright (C) 2018-2021 OpenCFD Ltd. + Copyright (C) 2018-2022 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. diff --git a/src/OpenFOAM/db/IOstreams/IOstreams/IOstreamOption.C b/src/OpenFOAM/db/IOstreams/IOstreams/IOstreamOption.C index a4d5255ee2..9be225b51c 100644 --- a/src/OpenFOAM/db/IOstreams/IOstreams/IOstreamOption.C +++ b/src/OpenFOAM/db/IOstreams/IOstreams/IOstreamOption.C @@ -26,7 +26,7 @@ License \*---------------------------------------------------------------------------*/ #include "IOstreamOption.H" -#include "error.H" +#include "debug.H" #include "dictionary.H" #include "Enum.H" #include "Switch.H" diff --git a/src/OpenFOAM/db/IOstreams/IOstreams/IOstreamOption.H b/src/OpenFOAM/db/IOstreams/IOstreams/IOstreamOption.H index 29bfd3c0b0..5b16cd3e3d 100644 --- a/src/OpenFOAM/db/IOstreams/IOstreams/IOstreamOption.H +++ b/src/OpenFOAM/db/IOstreams/IOstreams/IOstreamOption.H @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2011-2015 OpenFOAM Foundation - Copyright (C) 2018-2021 OpenCFD Ltd. + Copyright (C) 2018-2022 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -36,6 +36,9 @@ Description The compression (UNCOMPRESSED | COMPRESSED) is typically controlled by switch values (true/false, on/off, ...). + Additionally, some enumerations are defined (APPEND, NON_APPEND, ...) + that are useful, verbose alternatives to bool values. + SourceFiles IOstreamOption.C @@ -80,6 +83,20 @@ public: COMPRESSED //!< compression = true }; + //- File appending (NON_APPEND | APPEND) + enum appendType : char + { + NON_APPEND = 0, //!< append = false + APPEND //!< append = true + }; + + //- Atomic operations (output) + enum atomicType : char + { + NON_ATOMIC = 0, //!< atomic = false + ATOMIC //!< atomic = true + }; + //- Representation of a major/minor version number class versionNumber diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.C b/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.C index 9061999e69..3c2404bd02 100644 --- a/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.C +++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2017-2018 OpenFOAM Foundation - Copyright (C) 2019-2021 OpenCFD Ltd. + Copyright (C) 2019-2022 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -51,7 +51,7 @@ bool Foam::OFstreamCollator::writeFile const labelUList& recvSizes, const PtrList>& slaveData, // optional slave data IOstreamOption streamOpt, - const bool append, + IOstreamOption::appendType append, const dictionary& headerEntries ) { @@ -79,7 +79,7 @@ bool Foam::OFstreamCollator::writeFile osPtr.reset(new OFstream(fName, streamOpt, append)); auto& os = *osPtr; - if (!append) + if (append == IOstreamOption::NON_APPEND) { // No IOobject so cannot use IOobject::writeHeader @@ -348,7 +348,7 @@ bool Foam::OFstreamCollator::write const fileName& fName, const string& data, IOstreamOption streamOpt, - const bool append, + IOstreamOption::appendType append, const bool useThread, const dictionary& headerEntries ) diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.H b/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.H index dfcfdae236..3285e9eb7b 100644 --- a/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.H +++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.H @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2017-2018 OpenFOAM Foundation - Copyright (C) 2021 OpenCFD Ltd. + Copyright (C) 2021-2022 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -48,8 +48,8 @@ SourceFiles \*---------------------------------------------------------------------------*/ -#ifndef OFstreamCollator_H -#define OFstreamCollator_H +#ifndef Foam_OFstreamCollator_H +#define Foam_OFstreamCollator_H #include #include @@ -81,7 +81,7 @@ class OFstreamCollator const labelList sizes_; PtrList> slaveData_; const IOstreamOption streamOpt_; - const bool append_; + IOstreamOption::appendType append_; const dictionary headerEntries_; writeData @@ -92,7 +92,7 @@ class OFstreamCollator const string& data, const labelList& sizes, IOstreamOption streamOpt, - const bool append, + IOstreamOption::appendType append, const dictionary& headerEntries ) : @@ -157,7 +157,7 @@ class OFstreamCollator const labelUList& recvSizes, const PtrList>& slaveData, IOstreamOption streamOpt, - const bool append, + IOstreamOption::appendType append, const dictionary& headerEntries ); @@ -200,7 +200,7 @@ public: const fileName&, const string& data, IOstreamOption streamOpt, - const bool append, + IOstreamOption::appendType append, const bool useThread = true, const dictionary& headerEntries = dictionary::null ); diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C b/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C index 540163a01f..edfbec5563 100644 --- a/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C +++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C @@ -217,13 +217,17 @@ bool Foam::fileOperations::collatedFileOperation::appendObject // Note: cannot do append + compression. This is a limitation // of ogzstream (or rather most compressed formats) + // + // File should always be created as non-atomic + // (consistency between append/non-append) OFstream os ( pathName, - // UNCOMPRESSED + // UNCOMPRESSED (binary only) IOstreamOption(IOstreamOption::BINARY, streamOpt.version()), - !isMaster // append slaves + // Append on sub-ranks + (isMaster ? IOstreamOption::NON_APPEND : IOstreamOption::APPEND) ); if (!os.good()) @@ -381,7 +385,7 @@ bool Foam::fileOperations::collatedFileOperation::writeObject ( pathName, streamOpt, - false, // append=false + IOstreamOption::NON_APPEND, valid ); @@ -424,7 +428,7 @@ bool Foam::fileOperations::collatedFileOperation::writeObject ( pathName, streamOpt, - false, // append=false + IOstreamOption::NON_APPEND, valid ); diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/threadedCollatedOFstream.C b/src/OpenFOAM/global/fileOperations/collatedFileOperation/threadedCollatedOFstream.C index e0793a4307..bf8e046e74 100644 --- a/src/OpenFOAM/global/fileOperations/collatedFileOperation/threadedCollatedOFstream.C +++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/threadedCollatedOFstream.C @@ -59,7 +59,7 @@ Foam::threadedCollatedOFstream::~threadedCollatedOFstream() pathName_, str(), IOstreamOption(IOstreamOption::BINARY, version(), compression_), - false, // append=false + IOstreamOption::NON_APPEND, useThread_, headerEntries_ ); diff --git a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C index d6c071cd84..5205de0043 100644 --- a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C +++ b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C @@ -2455,7 +2455,7 @@ Foam::fileOperations::masterUncollatedFileOperation::NewOFstream ( pathName, streamOpt, - false, // append=false + IOstreamOption::NON_APPEND, valid ) ); diff --git a/src/fileFormats/ensight/file/ensightFile.C b/src/fileFormats/ensight/file/ensightFile.C index 4214b726df..bd39b34b0a 100644 --- a/src/fileFormats/ensight/file/ensightFile.C +++ b/src/fileFormats/ensight/file/ensightFile.C @@ -79,7 +79,7 @@ Foam::ensightFile::ensightFile IOstreamOption::streamFormat fmt ) : - OFstream(ensight::FileName(pathname), fmt) + OFstream(IOstreamOption::ATOMIC, ensight::FileName(pathname), fmt) { init(); } @@ -92,7 +92,7 @@ Foam::ensightFile::ensightFile IOstreamOption::streamFormat fmt ) : - OFstream(path/ensight::FileName(name), fmt) + OFstream(IOstreamOption::ATOMIC, path/ensight::FileName(name), fmt) { init(); } diff --git a/src/fileFormats/ensight/file/ensightFile.H b/src/fileFormats/ensight/file/ensightFile.H index ddf866c86d..6f63148fe2 100644 --- a/src/fileFormats/ensight/file/ensightFile.H +++ b/src/fileFormats/ensight/file/ensightFile.H @@ -92,7 +92,7 @@ public: // Static Functions //- Return a null ensightFile - inline static const ensightFile& null() + static const ensightFile& null() { return NullObjectRef(); } @@ -102,6 +102,7 @@ public: //- Construct from path-name. // The path-name is adjusted for valid ensight file naming. + // Always created as an atomic explicit ensightFile ( const fileName& pathname, @@ -110,6 +111,7 @@ public: //- Construct from path and name. // Only the name portion is adjusted for valid ensight file naming. + // Always created as an atomic ensightFile ( const fileName& path, diff --git a/src/fileFormats/ensight/file/ensightGeoFile.H b/src/fileFormats/ensight/file/ensightGeoFile.H index d18fbe0fcb..9fbc124902 100644 --- a/src/fileFormats/ensight/file/ensightGeoFile.H +++ b/src/fileFormats/ensight/file/ensightGeoFile.H @@ -67,7 +67,7 @@ public: // Static Functions //- Return a null ensightGeoFile - inline static const ensightGeoFile& null() + static const ensightGeoFile& null() { return NullObjectRef(); } diff --git a/src/lumpedPointMotion/movement/lumpedPointMovement.C b/src/lumpedPointMotion/movement/lumpedPointMovement.C index dcfc841ad2..5c23a55c3a 100644 --- a/src/lumpedPointMotion/movement/lumpedPointMovement.C +++ b/src/lumpedPointMotion/movement/lumpedPointMovement.C @@ -1314,7 +1314,7 @@ bool Foam::lumpedPointMovement::writeData ( coupler().resolveFile(logName_), IOstreamOption(), - true // append + IOstreamOption::APPEND ); writeData(os, forces, moments, outputFormatType::PLAIN, timesWritten);