diff --git a/applications/test/IOField/Test-IOField.C b/applications/test/IOField/Test-IOField.C index 6af6d81179..943e8314a8 100644 --- a/applications/test/IOField/Test-IOField.C +++ b/applications/test/IOField/Test-IOField.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2017 OpenFOAM Foundation - Copyright (C) 2019 OpenCFD Ltd. + Copyright (C) 2019-2021 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -96,8 +96,8 @@ void writeAndRead Pout<< "** Writing:" << writeType << " Reading:" << readType << endl; - autoPtr writeHandler(fileOperation::New(writeType, true)); - fileHandler(writeHandler); + // The write handler + fileHandler(fileOperation::New(writeType, true)); // Delete Pout<< "Deleting:" << fileHandler().filePath(io.objectPath()) << endl; @@ -107,8 +107,8 @@ void writeAndRead Pout<< "Writing:" << fileHandler().objectPath(io, io.name()) << endl; doWrite(io, sz); - autoPtr readHandler(fileOperation::New(readType, true)); - fileHandler(readHandler); + // The read handler + fileHandler(fileOperation::New(readType, true)); // Read IOobject readIO(io); @@ -130,8 +130,7 @@ void readIfPresent const word& readType ) { - autoPtr readHandler(fileOperation::New(readType, true)); - fileHandler(readHandler); + fileHandler(fileOperation::New(readType, true)); // Read Pout<< "Reading:" << fileHandler().filePath(io.objectPath()) << endl; @@ -178,6 +177,7 @@ void doTests(IOobject& io, const label sz) int main(int argc, char *argv[]) { + argList::noBanner(); argList::addBoolOption("bool", "Use bool for tests"); argList::addBoolOption("scalar", "Use scalar for tests"); argList::addBoolOption("label", "Use label for tests (default)"); @@ -209,6 +209,11 @@ int main(int argc, char *argv[]) IOobject::NO_WRITE ); + { + dictionary headerDict; + io.writeHeader(headerDict, "anything", IOstreamOption()); + Info<< "IOobjectHeader" << headerDict << nl; + } label tested = 0; diff --git a/src/OpenFOAM/Make/files b/src/OpenFOAM/Make/files index 5d41eddc04..7db14b3cff 100644 --- a/src/OpenFOAM/Make/files +++ b/src/OpenFOAM/Make/files @@ -311,6 +311,7 @@ $(IOdictionary)/unwatchedIOdictionary.C db/IOobjects/IOMap/IOMapName.C db/IOobjects/decomposedBlockData/decomposedBlockData.C +db/IOobjects/decomposedBlockData/decomposedBlockDataHeader.C db/IOobjects/GlobalIOField/GlobalIOFields.C @@ -320,6 +321,7 @@ db/IOobjects/GlobalIOList/globalIOLists.C IOobject = db/IOobject $(IOobject)/IOobject.C $(IOobject)/IOobjectIO.C +$(IOobject)/IOobjectMetaData.C $(IOobject)/IOobjectReadHeader.C $(IOobject)/IOobjectWriteHeader.C diff --git a/src/OpenFOAM/db/IOobject/IOobject.C b/src/OpenFOAM/db/IOobject/IOobject.C index 7b0928ebe6..5419a5abd6 100644 --- a/src/OpenFOAM/db/IOobject/IOobject.C +++ b/src/OpenFOAM/db/IOobject/IOobject.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2011-2017 OpenFOAM Foundation - Copyright (C) 2016-2020 OpenCFD Ltd. + Copyright (C) 2016-2021 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -28,7 +28,7 @@ License #include "IOobject.H" #include "Time.H" -#include "IFstream.H" +#include "Istream.H" // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * // @@ -37,6 +37,8 @@ namespace Foam defineTypeNameAndDebug(IOobject, 0); } +bool Foam::IOobject::bannerEnabled_(true); + char Foam::IOobject::scopeSeparator ( #ifdef _WIN32 @@ -47,7 +49,6 @@ char Foam::IOobject::scopeSeparator #endif ); - const Foam::Enum < Foam::IOobject::fileCheckTypes @@ -60,7 +61,6 @@ Foam::IOobject::fileCheckTypesNames { fileCheckTypes::inotifyMaster, "inotifyMaster" }, }); - // Default fileCheck type Foam::IOobject::fileCheckTypes Foam::IOobject::fileModificationChecking ( @@ -460,7 +460,7 @@ Foam::IOobject::IOobject // * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // -const Foam::objectRegistry& Foam::IOobject::db() const +const Foam::objectRegistry& Foam::IOobject::db() const noexcept { return db_; } diff --git a/src/OpenFOAM/db/IOobject/IOobject.H b/src/OpenFOAM/db/IOobject/IOobject.H index 409c503690..5877b1a2a2 100644 --- a/src/OpenFOAM/db/IOobject/IOobject.H +++ b/src/OpenFOAM/db/IOobject/IOobject.H @@ -64,6 +64,21 @@ Description - \par NO_WRITE No automatic write on destruction but can be written explicitly + When serializing, the IOobject characteristics are typically written + as a \c FoamFile header, which is a sub-dictionary with the following + type of content: + + \table + Property | Description | Type | Reqd | Deflt + version | The base format version | float | no | 2.0 + format | The stream format (ascii/binary) | word | yes | + arch | The architecture string | string | no | + note | Descriptive note about the object | string | no | + location | The relative location of the object | string | no | + class | The type of the object | word | yes | + object | The name of the object | word | yes | + \endtable + Note Specifying registered does not result in the IOobject itself being registered. It is only serves as guidance for a regIOobject using it. @@ -86,6 +101,7 @@ SourceFiles #include "fileName.H" #include "typeInfo.H" #include "autoPtr.H" +#include "IOstreamOption.H" #include "InfoProxy.H" #include "Enum.H" @@ -96,6 +112,7 @@ namespace Foam // Forward Declarations class Time; +class dictionary; class objectRegistry; /*---------------------------------------------------------------------------*\ @@ -146,6 +163,12 @@ public: private: + // Static Data Members + + //- Use an output file banner, enabled by default + static bool bannerEnabled_; + + // Private Data //- Name @@ -192,6 +215,27 @@ protected: // Protected Member Functions + //- Helper: write content for FoamFile IOobject header + //- with optional meta information. + static void writeHeaderContent + ( + Ostream& os, + const IOobject& io, + const word& objectType, + const dictionary* metaDataDict = nullptr + ); + + //- Helper: write dictionary content for FoamFile header + //- with optional meta information. + static void writeHeaderContent + ( + dictionary& os, + const IOobject& io, + const word& objectType, + IOstreamOption streamOpt, + const dictionary* metaDataDict = nullptr + ); + //- Set the object state to bad void setBad(const string& s); @@ -214,6 +258,21 @@ public: // Static Functions + //- Status of output file banner + static bool bannerEnabled() noexcept + { + return bannerEnabled_; + } + + //- Enable/disable an output file banner + // \return the previous value + static bool bannerEnabled(bool on) noexcept + { + bool old(bannerEnabled_); + bannerEnabled_ = on; + return old; + } + //- Split path into instance, local, name components // // The splitting behaviour is as follows: @@ -364,28 +423,28 @@ public: // Member Functions - // General access + // General Access //- Return the local objectRegistry - const objectRegistry& db() const; + const objectRegistry& db() const noexcept; //- Return Time associated with the objectRegistry const Time& time() const; //- Return name - inline const word& name() const; + inline const word& name() const noexcept; //- Return name of the class name read from header - inline const word& headerClassName() const; + inline const word& headerClassName() const noexcept; //- Return non-constant access to the class name read from header - inline word& headerClassName(); + inline word& headerClassName() noexcept; //- Return the optional note - inline const string& note() const; + inline const string& note() const noexcept; //- Return non-constant access to the optional note - inline string& note(); + inline string& note() noexcept; //- Rename virtual void rename(const word& newName) @@ -394,16 +453,16 @@ public: } //- Should object created with this IOobject be registered? - inline bool registerObject() const; + inline bool registerObject() const noexcept; //- Should object created with this IOobject be registered? - inline bool& registerObject(); + inline bool& registerObject() noexcept; //- Is object same for all processors? - inline bool globalObject() const; + inline bool globalObject() const noexcept; //- Is object same for all processors? - inline bool& globalObject(); + inline bool& globalObject() noexcept; //- The sizeof (label) in bytes, possibly read from the header inline unsigned labelByteSize() const noexcept; @@ -422,19 +481,25 @@ public: inline bool isHeaderClassName() const; + // Meta-data + + //- Return pointer to meta-data (if any) or nullptr + virtual const dictionary* findMetaData() const noexcept; + + // Read/write options //- The read option - inline readOption readOpt() const; + inline readOption readOpt() const noexcept; //- Non-constant access to the read option - inline readOption& readOpt(); + inline readOption& readOpt() noexcept; //- The write option - inline writeOption writeOpt() const; + inline writeOption writeOpt() const noexcept; //- Non-constant access to the write option - inline writeOption& writeOpt(); + inline writeOption& writeOpt() noexcept; // Path components @@ -449,11 +514,11 @@ public: const fileName& caseName() const; - inline const fileName& instance() const; + inline const fileName& instance() const noexcept; - inline fileName& instance(); + inline fileName& instance() noexcept; - inline const fileName& local() const; + inline const fileName& local() const noexcept; //- The complete path fileName path() const; @@ -489,9 +554,19 @@ public: // Reading - //- Read header + //- Parse 'FoamFile' header contents and set the IOobject + //- characteristics and return the stream characteristics. + IOstreamOption parseHeader(const dictionary& headerDict); + + //- Read header ('FoamFile' dictionary) and set the + //- IOobject and stream characteristics. bool readHeader(Istream& is); + //- Read header (the 'FoamFile' dictionary) and set the + //- IOobject and stream characteristics. + // Saves the header content in the given dictionary. + bool readHeader(dictionary& headerDict, Istream& is); + //- Read header (uses typeFilePath to find file) and check its info. // Optionally checks headerClassName against the type-name. // When search is false, simply use the current instance, @@ -521,24 +596,31 @@ public: //- Write the standard end file divider static Ostream& writeEndDivider(Ostream& os); - //- Write header with current type() and arch information + //- Write header with current type() bool writeHeader(Ostream& os) const; - //- Write header with override of type and optionally without - //- arch information in ASCII - bool writeHeader + //- Write header with override of type + bool writeHeader(Ostream& os, const word& objectType) const; + + //- Write header into a dictionary with current type() + //- and given output format + void writeHeader(dictionary& dict, IOstreamOption streamOpt) const; + + //- Write header into a dictionary with override of type + //- and given output format + void writeHeader ( - Ostream& os, + dictionary& dict, const word& objectType, - const bool noArchAscii = false + IOstreamOption streamOpt ) const; // Error Handling - inline bool good() const; + inline bool good() const noexcept; - inline bool bad() const; + inline bool bad() const noexcept; // Info diff --git a/src/OpenFOAM/db/IOobject/IOobjectI.H b/src/OpenFOAM/db/IOobject/IOobjectI.H index 345ca59dd6..2bbb65836b 100644 --- a/src/OpenFOAM/db/IOobject/IOobjectI.H +++ b/src/OpenFOAM/db/IOobject/IOobjectI.H @@ -67,7 +67,7 @@ inline Foam::word Foam::IOobject::scopedName // General access -inline const Foam::word& Foam::IOobject::name() const +inline const Foam::word& Foam::IOobject::name() const noexcept { return name_; } @@ -85,49 +85,49 @@ inline Foam::word Foam::IOobject::member() const } -inline const Foam::word& Foam::IOobject::headerClassName() const +inline const Foam::word& Foam::IOobject::headerClassName() const noexcept { return headerClassName_; } -inline Foam::word& Foam::IOobject::headerClassName() +inline Foam::word& Foam::IOobject::headerClassName() noexcept { return headerClassName_; } -inline const Foam::string& Foam::IOobject::note() const +inline const Foam::string& Foam::IOobject::note() const noexcept { return note_; } -inline Foam::string& Foam::IOobject::note() +inline Foam::string& Foam::IOobject::note() noexcept { return note_; } -inline bool Foam::IOobject::registerObject() const +inline bool Foam::IOobject::registerObject() const noexcept { return registerObject_; } -inline bool& Foam::IOobject::registerObject() +inline bool& Foam::IOobject::registerObject() noexcept { return registerObject_; } -inline bool Foam::IOobject::globalObject() const +inline bool Foam::IOobject::globalObject() const noexcept { return globalObject_; } -inline bool& Foam::IOobject::globalObject() +inline bool& Foam::IOobject::globalObject() noexcept { return globalObject_; } @@ -162,25 +162,25 @@ inline bool Foam::IOobject::isHeaderClassName() const // Read/write options -inline Foam::IOobject::readOption Foam::IOobject::readOpt() const +inline Foam::IOobject::readOption Foam::IOobject::readOpt() const noexcept { return rOpt_; } -inline Foam::IOobject::readOption& Foam::IOobject::readOpt() +inline Foam::IOobject::readOption& Foam::IOobject::readOpt() noexcept { return rOpt_; } -inline Foam::IOobject::writeOption Foam::IOobject::writeOpt() const +inline Foam::IOobject::writeOption Foam::IOobject::writeOpt() const noexcept { return wOpt_; } -inline Foam::IOobject::writeOption& Foam::IOobject::writeOpt() +inline Foam::IOobject::writeOption& Foam::IOobject::writeOpt() noexcept { return wOpt_; } @@ -188,19 +188,19 @@ inline Foam::IOobject::writeOption& Foam::IOobject::writeOpt() // Path components -inline const Foam::fileName& Foam::IOobject::instance() const +inline const Foam::fileName& Foam::IOobject::instance() const noexcept { return instance_; } -inline Foam::fileName& Foam::IOobject::instance() +inline Foam::fileName& Foam::IOobject::instance() noexcept { return instance_; } -inline const Foam::fileName& Foam::IOobject::local() const +inline const Foam::fileName& Foam::IOobject::local() const noexcept { return local_; } @@ -214,13 +214,13 @@ inline Foam::fileName Foam::IOobject::objectPath() const // Error Handling -inline bool Foam::IOobject::good() const +inline bool Foam::IOobject::good() const noexcept { return objState_ == GOOD; } -inline bool Foam::IOobject::bad() const +inline bool Foam::IOobject::bad() const noexcept { return objState_ == BAD; } diff --git a/src/OpenFOAM/db/IOobject/IOobjectMetaData.C b/src/OpenFOAM/db/IOobject/IOobjectMetaData.C new file mode 100644 index 0000000000..260186dc9d --- /dev/null +++ b/src/OpenFOAM/db/IOobject/IOobjectMetaData.C @@ -0,0 +1,39 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | OpenFOAM: The Open Source CFD Toolbox + \\ / O peration | + \\ / A nd | www.openfoam.com + \\/ M anipulation | +------------------------------------------------------------------------------- + Copyright (C) 2021 OpenCFD Ltd. +------------------------------------------------------------------------------- +License + This file is part of OpenFOAM. + + OpenFOAM is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with OpenFOAM. If not, see . + +\*---------------------------------------------------------------------------*/ + +#include "IOobject.H" +#include "dictionary.H" + +// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // + +const Foam::dictionary* Foam::IOobject::findMetaData() const noexcept +{ + return nullptr; +} + + +// ************************************************************************* // diff --git a/src/OpenFOAM/db/IOobject/IOobjectReadHeader.C b/src/OpenFOAM/db/IOobject/IOobjectReadHeader.C index ed0ea688a4..04e9dac79b 100644 --- a/src/OpenFOAM/db/IOobject/IOobjectReadHeader.C +++ b/src/OpenFOAM/db/IOobject/IOobjectReadHeader.C @@ -32,7 +32,45 @@ License // * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // -bool Foam::IOobject::readHeader(Istream& is) +Foam::IOstreamOption Foam::IOobject::parseHeader(const dictionary& headerDict) +{ + IOstreamOption streamOpt; // == (ASCII, currentVersion) + + // Treat "version" as optional + { + token tok; + if (headerDict.readIfPresent("version", tok)) + { + streamOpt.version(tok); + } + } + + // Treat "format" as mandatory, could also as optional + streamOpt.format(headerDict.get("format")); + + headerClassName_ = headerDict.get("class"); + + const word headerObject(headerDict.get("object")); + + // The "note" entry is optional + headerDict.readIfPresent("note", note_); + + // The "arch" information may be missing + string arch; + if (headerDict.readIfPresent("arch", arch)) + { + unsigned val = foamVersion::labelByteSize(arch); + if (val) sizeofLabel_ = static_cast(val); + + val = foamVersion::scalarByteSize(arch); + if (val) sizeofScalar_ = static_cast(val); + } + + return streamOpt; +} + + +bool Foam::IOobject::readHeader(dictionary& headerDict, Istream& is) { if (IOobject::debug) { @@ -64,48 +102,21 @@ bool Foam::IOobject::readHeader(Istream& is) if (is.good() && firstToken.isWord("FoamFile")) { - const dictionary headerDict(is); + headerDict.read(is, false); // Read sub-dictionary content - is.version(headerDict.get("version")); - is.format(headerDict.get("format")); - - headerClassName_ = headerDict.get("class"); - - const word headerObject(headerDict.get("object")); - - if (IOobject::debug && headerObject != name()) - { - IOWarningInFunction(is) - << " object renamed from " - << name() << " to " << headerObject - << " for file " << is.name() << endl; - } - - // The note entry is optional - headerDict.readIfPresent("note", note_); - - sizeofLabel_ = sizeof(label); - sizeofScalar_ = sizeof(scalar); - - // The arch information is optional - string arch; - if (headerDict.readIfPresent("arch", arch)) - { - unsigned val = foamVersion::labelByteSize(arch); - if (val) sizeofLabel_ = static_cast(val); - - val = foamVersion::scalarByteSize(arch); - if (val) sizeofScalar_ = static_cast(val); - } + IOstreamOption streamOpt = parseHeader(headerDict); + is.format(streamOpt.format()); + is.version(streamOpt.version()); is.setLabelByteSize(sizeofLabel_); is.setScalarByteSize(sizeofScalar_); } else { IOWarningInFunction(is) - << "First token could not be read or is not the keyword 'FoamFile'" - << nl << nl << "Check header is of the form:" << nl << endl; + << "First token could not be read or is not 'FoamFile'" + << nl << nl + << "Check header is of the form:" << nl << endl; writeHeader(Info); @@ -151,4 +162,11 @@ bool Foam::IOobject::readHeader(Istream& is) } +bool Foam::IOobject::readHeader(Istream& is) +{ + dictionary headerDict; + return IOobject::readHeader(headerDict, is); +} + + // ************************************************************************* // diff --git a/src/OpenFOAM/db/IOobject/IOobjectWriteHeader.C b/src/OpenFOAM/db/IOobject/IOobjectWriteHeader.C index 2ea386443e..c66ed91a1e 100644 --- a/src/OpenFOAM/db/IOobject/IOobjectWriteHeader.C +++ b/src/OpenFOAM/db/IOobject/IOobjectWriteHeader.C @@ -27,10 +27,40 @@ License \*---------------------------------------------------------------------------*/ #include "IOobject.H" +#include "dictionary.H" #include "objectRegistry.H" #include "foamVersion.H" -// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // +// * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * // + +namespace Foam +{ + +inline void writeSpaces(Ostream& os, label nSpaces) +{ + if (nSpaces < 1) + { + nSpaces = 1; + } + while (nSpaces--) + { + os.write(char(token::SPACE)); + } +} + +// Similar to writeEntry, but with fewer spaces +template +inline void writeHeaderEntry(Ostream& os, const word& key, const T& value) +{ + os << indent << key; + writeSpaces(os, 12 - label(key.size())); + os << value << char(token::END_STATEMENT) << nl; +} + +} // End namespace Foam + + +// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * // // A banner corresponding to this: // @@ -117,11 +147,93 @@ Foam::Ostream& Foam::IOobject::writeEndDivider(Ostream& os) } +// * * * * * * * * * * * * Protected Member Functions * * * * * * * * * * * // + +void Foam::IOobject::writeHeaderContent +( + Ostream& os, + const IOobject& io, + const word& objectType, + const dictionary* metaDataDict +) +{ + // Standard header entries + writeHeaderEntry(os, "version", os.version()); + writeHeaderEntry(os, "format", os.format()); + writeHeaderEntry(os, "arch", foamVersion::buildArch); + + if (!io.note().empty()) + { + writeHeaderEntry(os, "note", io.note()); + } + + if (objectType.empty()) + { + // Empty type not allowed - use 'dictionary' fallback + writeHeaderEntry(os, "class", word("dictionary")); + } + else + { + writeHeaderEntry(os, "class", objectType); + } + + writeHeaderEntry(os, "location", io.instance()/io.db().dbDir()/io.local()); + writeHeaderEntry(os, "object", io.name()); + + // Meta-data (if any) + if (metaDataDict && !metaDataDict->empty()) + { + metaDataDict->writeEntry("meta", os); + } +} + + +void Foam::IOobject::writeHeaderContent +( + dictionary& dict, + const IOobject& io, + const word& objectType, + IOstreamOption streamOpt, + const dictionary* metaDataDict +) +{ + // Standard header entries + dict.set("version", streamOpt.version()); + dict.set("format", streamOpt.format()); + dict.set("arch", foamVersion::buildArch); + + if (!io.note().empty()) + { + dict.set("note", io.note()); + } + + if (objectType.empty()) + { + // Empty type not allowed - use 'dictionary' fallback + dict.set("class", word("dictionary")); + } + else + { + dict.set("class", objectType); + } + + dict.set("location", io.instance()/io.db().dbDir()/io.local()); + dict.set("object", io.name()); + + // Deep-copy of meta-data (if any) + if (metaDataDict && !metaDataDict->empty()) + { + dict.add("meta", *metaDataDict); + } +} + + +// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // + bool Foam::IOobject::writeHeader ( Ostream& os, - const word& objectType, - const bool noArchAscii + const word& objectType ) const { if (!os.good()) @@ -133,39 +245,28 @@ bool Foam::IOobject::writeHeader return false; } - IOobject::writeBanner(os) - << "FoamFile" << nl - << '{' << nl - << " version " << os.version() << ';' << nl - << " format " << os.format() << ';' << nl; - - if (os.format() == IOstream::BINARY || !noArchAscii) + if (IOobject::bannerEnabled()) { - // Arch information (BINARY: always, ASCII: can disable) - os << " arch " << foamVersion::buildArch << ';' << nl; - } - if (!note().empty()) - { - os << " note " << note() << ';' << nl; + IOobject::writeBanner(os); } - os << " class "; - if (objectType.empty()) - { - // Empty type not allowed - use 'dictionary' fallback - os << "dictionary"; - } - else - { - os << objectType; - } - os << ';' << nl; + os.beginBlock("FoamFile"); - os << " location " << instance()/db().dbDir()/local() << ';' << nl - << " object " << name() << ';' << nl - << '}' << nl; + // Standard header entries + IOobject::writeHeaderContent + ( + os, + *this, + objectType, + this->findMetaData() + ); - writeDivider(os) << nl; + os.endBlock(); + + if (IOobject::bannerEnabled()) + { + IOobject::writeDivider(os) << nl; + } return true; } @@ -173,7 +274,35 @@ bool Foam::IOobject::writeHeader bool Foam::IOobject::writeHeader(Ostream& os) const { - return writeHeader(os, type()); + return IOobject::writeHeader(os, this->type()); +} + + +void Foam::IOobject::writeHeader +( + dictionary& dict, + const word& objectType, + IOstreamOption streamOpt +) const +{ + IOobject::writeHeaderContent + ( + dict, + *this, + objectType, + streamOpt, + this->findMetaData() + ); +} + + +void Foam::IOobject::writeHeader +( + dictionary& dict, + IOstreamOption streamOpt +) const +{ + IOobject::writeHeader(dict, this->type(), streamOpt); } diff --git a/src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.C b/src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.C index 89affe61e0..379ef7fdd2 100644 --- a/src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.C +++ b/src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.C @@ -34,10 +34,11 @@ License #include "dictionary.H" #include "objectRegistry.H" #include "SubList.H" +#include "charList.H" #include "labelPair.H" #include "masterUncollatedFileOperation.H" -#include "IListStream.H" -#include "foamVersion.H" +#include "ListStream.H" +#include "StringStream.H" // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * // @@ -46,6 +47,30 @@ namespace Foam defineTypeNameAndDebug(decomposedBlockData, 0); } + +// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * // + +bool Foam::decomposedBlockData::isCollatedType +( + const word& objectType +) +{ + return + ( + objectType == decomposedBlockData::typeName + ); +} + + +bool Foam::decomposedBlockData::isCollatedType +( + const IOobject& io +) +{ + return decomposedBlockData::isCollatedType(io.headerClassName()); +} + + // * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * // Foam::decomposedBlockData::decomposedBlockData @@ -57,7 +82,8 @@ Foam::decomposedBlockData::decomposedBlockData : regIOobject(io), commsType_(commsType), - comm_(comm) + comm_(comm), + contentData_() { // Temporary warning if (io.readOpt() == IOobject::MUST_READ_IF_MODIFIED) @@ -82,161 +108,125 @@ Foam::decomposedBlockData::decomposedBlockData } -Foam::decomposedBlockData::decomposedBlockData -( - const label comm, - const IOobject& io, - const UList& list, - const UPstream::commsTypes commsType -) -: - regIOobject(io), - commsType_(commsType), - comm_(comm) -{ - // Temporary warning - if (io.readOpt() == IOobject::MUST_READ_IF_MODIFIED) - { - WarningInFunction - << "decomposedBlockData " << name() - << " constructed with IOobject::MUST_READ_IF_MODIFIED" - " but decomposedBlockData does not support automatic rereading." - << endl; - } - - if - ( - ( - io.readOpt() == IOobject::MUST_READ - || io.readOpt() == IOobject::MUST_READ_IF_MODIFIED - ) - || (io.readOpt() == IOobject::READ_IF_PRESENT && headerOk()) - ) - { - read(); - } - else - { - List::operator=(list); - } -} - - -Foam::decomposedBlockData::decomposedBlockData -( - const label comm, - const IOobject& io, - List&& list, - const UPstream::commsTypes commsType -) -: - regIOobject(io), - commsType_(commsType), - comm_(comm) -{ - // Temporary warning - if (io.readOpt() == IOobject::MUST_READ_IF_MODIFIED) - { - WarningInFunction - << "decomposedBlockData " << name() - << " constructed with IOobject::MUST_READ_IF_MODIFIED" - " but decomposedBlockData does not support automatic rereading." - << endl; - } - - List::transfer(list); - - if - ( - ( - io.readOpt() == IOobject::MUST_READ - || io.readOpt() == IOobject::MUST_READ_IF_MODIFIED - ) - || (io.readOpt() == IOobject::READ_IF_PRESENT && headerOk()) - ) - { - read(); - } -} - - // * * * * * * * * * * * * * * * Members Functions * * * * * * * * * * * * * // -bool Foam::decomposedBlockData::readMasterHeader(IOobject& io, Istream& is) -{ - if (debug) - { - Pout<< "decomposedBlockData::readMasterHeader:" - << " stream:" << is.name() << endl; - } - - // Master-only reading of header - is.fatalCheck("read(Istream&)"); - - List data(is); - is.fatalCheck("read(Istream&) : reading entry"); - - UIListStream headerStream(data); - headerStream.name() = is.name(); - - return io.readHeader(headerStream); -} - - -void Foam::decomposedBlockData::writeHeader +bool Foam::decomposedBlockData::readBlockEntry ( - Ostream& os, - IOstreamOption streamOpt, - const word& objectType, - const string& note, - const fileName& location, - const word& objectName + Istream& is, + List& charData ) { - IOobject::writeBanner(os) - << "FoamFile" << nl - << '{' << nl - << " version " << streamOpt.version() << ';' << nl - << " format " << streamOpt.format() << ';' << nl - << " arch " << foamVersion::buildArch << ';' << nl; + // Handle any of these: - if (Pstream::parRun()) + // 0. NCHARS (...) + // 1. List NCHARS (...) + // 2. processorN List NCHARS (...) ; + + is.fatalCheck(FUNCTION_NAME); + token tok(is); + is.fatalCheck(FUNCTION_NAME); + + // Dictionary format has primitiveEntry keyword: + const bool isDictFormat = (tok.isWord() && !tok.isCompound()); + + if (!isDictFormat && tok.good()) { - os << " blocks " << Pstream::nProcs() << ';' << nl; + is.putBack(tok); } - if (!note.empty()) + charData.readList(is); + + if (isDictFormat) { - os << " note " << note << ';' << nl; + is.fatalCheck(FUNCTION_NAME); + is >> tok; + is.fatalCheck(FUNCTION_NAME); + + // Swallow trailing ';' + if (tok.good() && !tok.isPunctuation(token::END_STATEMENT)) + { + is.putBack(tok); + } } - os << " class "; - if (objectType.empty()) - { - // Empty type not allowed - use 'dictionary' fallback - os << "dictionary"; - } - else - { - os << objectType; - } - os << ';' << nl; - - if (!location.empty()) - { - os << " location " << location << ';' << nl; - } - - os << " object " << objectName << ';' << nl - << '}' << nl; - - IOobject::writeDivider(os) << nl; + return true; } -Foam::autoPtr Foam::decomposedBlockData::readBlock +std::streamoff Foam::decomposedBlockData::writeBlockEntry +( + OSstream& os, + const label blocki, + const UList& charData +) +{ + // Offset to the beginning of this output + + std::streamoff blockOffset = os.stdStream().tellp(); + + const word procName("processor" + Foam::name(blocki)); + + { + os << nl << "// " << procName << nl; + charData.writeList(os) << nl; + } + + return blockOffset; +} + + +std::streamoff Foam::decomposedBlockData::writeBlockEntry +( + OSstream& os, + IOstreamOption streamOptData, + const regIOobject& io, + const label blocki, + const bool withLocalHeader +) +{ + // String(s) from all data to write + string contentChars; + { + OStringStream os(streamOptData); + + bool ok = true; + + // Generate FoamFile header on master, without comment banner + if (withLocalHeader) + { + const bool old = IOobject::bannerEnabled(false); + + ok = io.writeHeader(os); + + IOobject::bannerEnabled(old); + } + + // Write the data to the Ostream + ok = ok && io.writeData(os); + + if (!ok) + { + return std::streamoff(-1); + } + + contentChars = os.str(); + } + + // The character data + UList charData + ( + const_cast(contentChars.data()), + label(contentChars.size()) + ); + + return decomposedBlockData::writeBlockEntry(os, blocki, charData); +} + + +Foam::autoPtr +Foam::decomposedBlockData::readBlock ( const label blocki, - Istream& is, + ISstream& is, IOobject& headerIO ) { @@ -247,66 +237,64 @@ Foam::autoPtr Foam::decomposedBlockData::readBlock << endl; } - is.fatalCheck("read(Istream&)"); + // Extracted header information + IOstreamOption streamOptData; + unsigned labelWidth = is.labelByteSize(); + unsigned scalarWidth = is.scalarByteSize(); autoPtr realIsPtr; + // Read master for header + List data; + decomposedBlockData::readBlockEntry(is, data); + if (blocki == 0) { - List data(is); - is.fatalCheck("read(Istream&) : reading entry"); - realIsPtr.reset(new IListStream(std::move(data))); realIsPtr->name() = is.name(); - // Read header - if (!headerIO.readHeader(*realIsPtr)) { - FatalIOErrorInFunction(*realIsPtr) - << "problem while reading header for object " - << is.name() << exit(FatalIOError); + // Read header from first block, + // advancing the stream position + if (!headerIO.readHeader(*realIsPtr)) + { + FatalIOErrorInFunction(*realIsPtr) + << "Problem while reading header for object " + << is.name() << nl + << exit(FatalIOError); + } } } else { - // Read master for header - List data(is); - is.fatalCheck("read(Istream&) : reading entry"); - - IOstreamOption::versionNumber ver(IOstreamOption::currentVersion); - IOstreamOption::streamFormat fmt; - unsigned labelByteSize; - unsigned scalarByteSize; { + // Read header from first block UIListStream headerStream(data); - - // Read header if (!headerIO.readHeader(headerStream)) { FatalIOErrorInFunction(headerStream) - << "problem while reading header for object " - << is.name() << exit(FatalIOError); + << "Problem while reading header for object " + << is.name() << nl + << exit(FatalIOError); } - ver = headerStream.version(); - fmt = headerStream.format(); - labelByteSize = headerStream.labelByteSize(); - scalarByteSize = headerStream.scalarByteSize(); + streamOptData = static_cast(headerStream); + labelWidth = headerStream.labelByteSize(); + scalarWidth = headerStream.scalarByteSize(); } for (label i = 1; i < blocki+1; i++) { // Read and discard data, only retain the last one - is >> data; - is.fatalCheck("read(Istream&) : reading entry"); + decomposedBlockData::readBlockEntry(is, data); } realIsPtr.reset(new IListStream(std::move(data))); realIsPtr->name() = is.name(); - // Apply master stream settings to realIsPtr - realIsPtr().format(fmt); - realIsPtr().version(ver); - realIsPtr().setLabelByteSize(labelByteSize); - realIsPtr().setScalarByteSize(scalarByteSize); + // Apply stream settings + realIsPtr().format(streamOptData.format()); + realIsPtr().version(streamOptData.version()); + realIsPtr().setLabelByteSize(labelWidth); + realIsPtr().setScalarByteSize(scalarWidth); } return realIsPtr; @@ -331,24 +319,28 @@ bool Foam::decomposedBlockData::readBlocks bool ok = false; + if (UPstream::master(comm)) + { + auto& is = *isPtr; + is.fatalCheck(FUNCTION_NAME); + + // Read master data + decomposedBlockData::readBlockEntry(is, data); + } + if (commsType == UPstream::commsTypes::scheduled) { if (UPstream::master(comm)) { - Istream& is = *isPtr; - is.fatalCheck("read(Istream&)"); + // Master data already read ... + auto& is = *isPtr; + is.fatalCheck(FUNCTION_NAME); - // Read master data - { - is >> data; - is.fatalCheck("read(Istream&) : reading entry"); - } - - // Read slave data + // Read and transmit slave data for (const int proci : UPstream::subProcs(comm)) { - List elems(is); - is.fatalCheck("read(Istream&) : reading entry"); + List elems; + decomposedBlockData::readBlockEntry(is, elems); OPstream os ( @@ -387,20 +379,15 @@ bool Foam::decomposedBlockData::readBlocks if (UPstream::master(comm)) { - Istream& is = *isPtr; - is.fatalCheck("read(Istream&)"); + // Master data already read ... + auto& is = *isPtr; + is.fatalCheck(FUNCTION_NAME); - // Read master data - { - is >> data; - is.fatalCheck("read(Istream&) : reading entry"); - } - - // Read slave data + // Read and transmit slave data for (const int proci : UPstream::subProcs(comm)) { - List elems(is); - is.fatalCheck("read(Istream&) : reading entry"); + List elems; + decomposedBlockData::readBlockEntry(is, elems); UOPstream os(proci, pBufs); os << elems; @@ -440,39 +427,45 @@ Foam::autoPtr Foam::decomposedBlockData::readBlocks } bool ok = false; - List data; autoPtr realIsPtr; + if (UPstream::master(comm)) + { + auto& is = *isPtr; + is.fatalCheck(FUNCTION_NAME); + + // Read master data + decomposedBlockData::readBlockEntry(is, data); + + realIsPtr.reset(new IListStream(std::move(data))); + realIsPtr->name() = fName; + + { + // Read header from first block, + // advancing the stream position + if (!headerIO.readHeader(*realIsPtr)) + { + FatalIOErrorInFunction(*realIsPtr) + << "Problem while reading header for object " + << is.name() << nl + << exit(FatalIOError); + } + } + } + if (commsType == UPstream::commsTypes::scheduled) { if (UPstream::master(comm)) { - Istream& is = *isPtr; - is.fatalCheck("read(Istream&)"); + // Master data already read ... + auto& is = *isPtr; + is.fatalCheck(FUNCTION_NAME); - // Read master data - { - is >> data; - is.fatalCheck("read(Istream&) : reading entry"); - - realIsPtr.reset(new IListStream(std::move(data))); - realIsPtr->name() = fName; - - // Read header - if (!headerIO.readHeader(*realIsPtr)) - { - FatalIOErrorInFunction(*realIsPtr) - << "problem while reading header for object " - << is.name() << exit(FatalIOError); - } - } - - // Read slave data + // Read and transmit slave data for (const int proci : UPstream::subProcs(comm)) { - is >> data; - is.fatalCheck("read(Istream&) : reading entry"); + decomposedBlockData::readBlockEntry(is, data); OPstream os ( @@ -514,31 +507,15 @@ Foam::autoPtr Foam::decomposedBlockData::readBlocks if (UPstream::master(comm)) { - Istream& is = *isPtr; - is.fatalCheck("read(Istream&)"); + // Master data already read ... + auto& is = *isPtr; + is.fatalCheck(FUNCTION_NAME); - // Read master data - { - is >> data; - is.fatalCheck("read(Istream&) : reading entry"); - - realIsPtr.reset(new IListStream(std::move(data))); - realIsPtr->name() = fName; - - // Read header - if (!headerIO.readHeader(*realIsPtr)) - { - FatalIOErrorInFunction(*realIsPtr) - << "problem while reading header for object " - << is.name() << exit(FatalIOError); - } - } - - // Read slave data + // Read and transmit slave data for (const int proci : UPstream::subProcs(comm)) { - List elems(is); - is.fatalCheck("read(Istream&) : reading entry"); + List elems; + decomposedBlockData::readBlockEntry(is, elems); UOPstream os(proci, pBufs); os << elems; @@ -567,24 +544,24 @@ Foam::autoPtr Foam::decomposedBlockData::readBlocks // Scatter master header info int verValue; int fmtValue; - unsigned labelByteSize; - unsigned scalarByteSize; + unsigned labelWidth; + unsigned scalarWidth; if (UPstream::master(comm)) { verValue = realIsPtr().version().canonical(); fmtValue = static_cast(realIsPtr().format()); - labelByteSize = realIsPtr().labelByteSize(); - scalarByteSize = realIsPtr().scalarByteSize(); + labelWidth = realIsPtr().labelByteSize(); + scalarWidth = realIsPtr().scalarByteSize(); } Pstream::scatter(verValue); //, Pstream::msgType(), comm); Pstream::scatter(fmtValue); //, Pstream::msgType(), comm); - Pstream::scatter(labelByteSize); //, Pstream::msgType(), comm); - Pstream::scatter(scalarByteSize); //, Pstream::msgType(), comm); + Pstream::scatter(labelWidth); //, Pstream::msgType(), comm); + Pstream::scatter(scalarWidth); //, Pstream::msgType(), comm); realIsPtr().version(IOstreamOption::versionNumber::canonical(verValue)); realIsPtr().format(IOstreamOption::streamFormat(fmtValue)); - realIsPtr().setLabelByteSize(labelByteSize); - realIsPtr().setScalarByteSize(scalarByteSize); + realIsPtr().setLabelByteSize(labelWidth); + realIsPtr().setScalarByteSize(scalarWidth); word name(headerIO.name()); Pstream::scatter(name, Pstream::msgType(), comm); @@ -656,8 +633,8 @@ void Foam::decomposedBlockData::gatherSlaveData { const label numProcs = UPstream::nProcs(comm); - sliceSizes.setSize(numProcs, 0); - sliceOffsets.setSize(numProcs+1, 0); + sliceSizes.resize(numProcs, 0); + sliceOffsets.resize(numProcs+1, 0); int totalSize = 0; label proci = startProc; @@ -741,8 +718,8 @@ bool Foam::decomposedBlockData::writeBlocks ( const label comm, autoPtr& osPtr, - List& start, - const UList& data, + List& blockOffset, + const UList& masterData, const labelUList& recvSizes, const PtrList>& slaveData, @@ -755,7 +732,7 @@ bool Foam::decomposedBlockData::writeBlocks { Pout<< "decomposedBlockData::writeBlocks:" << " stream:" << (osPtr ? osPtr->name() : "invalid") - << " data:" << data.size() + << " data:" << masterData.size() << " (master only) slaveData:" << slaveData.size() << " commsType:" << Pstream::commsTypeNames[commsType] << endl; } @@ -764,35 +741,43 @@ bool Foam::decomposedBlockData::writeBlocks bool ok = true; + // Write master data + if (UPstream::master(comm)) + { + blockOffset.resize(nProcs); + + OSstream& os = *osPtr; + + blockOffset[UPstream::masterNo()] = + decomposedBlockData::writeBlockEntry + ( + os, + UPstream::masterNo(), + masterData + ); + + ok = os.good(); + } + if (slaveData.size()) { - // Already have gathered the slave data. communicator only used to - // check who is the master + // Already have gathered the slave data. if (UPstream::master(comm)) { + // Master data already written ... OSstream& os = *osPtr; - start.setSize(nProcs); - - // Write master data - { - os << nl << "// Processor" << UPstream::masterNo() << nl; - start[UPstream::masterNo()] = os.stdStream().tellp(); - os << data; - } - // Write slaves - - label slaveOffset = 0; - for (label proci = 1; proci < nProcs; ++proci) { - os << nl << nl << "// Processor" << proci << nl; - start[proci] = os.stdStream().tellp(); - - os << slaveData[proci]; - slaveOffset += recvSizes[proci]; + blockOffset[proci] = + decomposedBlockData::writeBlockEntry + ( + os, + proci, + slaveData[proci] + ); } ok = os.good(); @@ -802,21 +787,14 @@ bool Foam::decomposedBlockData::writeBlocks { if (UPstream::master(comm)) { - start.setSize(nProcs); - + // Master data already written ... OSstream& os = *osPtr; - // Write master data - { - os << nl << "// Processor" << UPstream::masterNo() << nl; - start[UPstream::masterNo()] = os.stdStream().tellp(); - os << data; - } - // Write slaves - List elems; + // Receive and write slaves + DynamicList elems; for (label proci = 1; proci < nProcs; ++proci) { - elems.setSize(recvSizes[proci]); + elems.resize(recvSizes[proci]); IPstream::read ( UPstream::commsTypes::scheduled, @@ -827,9 +805,13 @@ bool Foam::decomposedBlockData::writeBlocks comm ); - os << nl << nl << "// Processor" << proci << nl; - start[proci] = os.stdStream().tellp(); - os << elems; + blockOffset[proci] = + decomposedBlockData::writeBlockEntry + ( + os, + proci, + elems + ); } ok = os.good(); @@ -840,8 +822,8 @@ bool Foam::decomposedBlockData::writeBlocks ( UPstream::commsTypes::scheduled, UPstream::masterNo(), - data.cdata(), - data.size_bytes(), + masterData.cdata(), + masterData.size_bytes(), Pstream::msgType(), comm ); @@ -849,18 +831,7 @@ bool Foam::decomposedBlockData::writeBlocks } else { - // Write master data - if (UPstream::master(comm)) - { - start.setSize(nProcs); - - OSstream& os = *osPtr; - - os << nl << "// Processor" << UPstream::masterNo() << nl; - start[UPstream::masterNo()] = os.stdStream().tellp(); - os << data; - } - + // Master data already written ... // Find out how many processor can be received into // maxMasterFileBufferSize @@ -895,11 +866,11 @@ bool Foam::decomposedBlockData::writeBlocks gatherSlaveData ( comm, - data, + masterData, recvSizes, - startProc, // startProc, - nSendProcs, // nProcs, + startProc, // startProc, + nSendProcs, // nProcs, sliceOffsets, recvData @@ -914,18 +885,22 @@ bool Foam::decomposedBlockData::writeBlocks ( label proci = startProc; proci < startProc+nSendProcs; - proci++ + ++proci ) { - os << nl << nl << "// Processor" << proci << nl; - start[proci] = os.stdStream().tellp(); + SubList dataSlice + ( + recvData, + sliceOffsets[proci+1]-sliceOffsets[proci], + sliceOffsets[proci] + ); - os << - SubList + blockOffset[proci] = + decomposedBlockData::writeBlockEntry ( - recvData, - sliceOffsets[proci+1]-sliceOffsets[proci], - sliceOffsets[proci] + os, + proci, + dataSlice ); } } @@ -960,15 +935,12 @@ bool Foam::decomposedBlockData::read() IOobject::readHeader(*isPtr); } - List& data = *this; - return readBlocks(comm_, isPtr, data, commsType_); + return readBlocks(comm_, isPtr, contentData_, commsType_); } bool Foam::decomposedBlockData::writeData(Ostream& os) const { - const List& data = *this; - IOobject io(*this); IOstreamOption streamOpt(os); @@ -978,7 +950,7 @@ bool Foam::decomposedBlockData::writeData(Ostream& os) const // Re-read my own data to find out the header information if (Pstream::master(comm_)) { - UIListStream headerStream(data); + UIListStream headerStream(contentData_); io.readHeader(headerStream); verValue = headerStream.version().canonical(); @@ -1008,7 +980,7 @@ bool Foam::decomposedBlockData::writeData(Ostream& os) const decomposedBlockData::writeHeader ( os, - streamOpt, + streamOpt, // streamOpt for data io.headerClassName(), io.note(), masterLocation, @@ -1020,12 +992,12 @@ bool Foam::decomposedBlockData::writeData(Ostream& os) const if (isA(os)) { // Serial file output - can use writeRaw() - os.writeRaw(data.cdata(), data.size_bytes()); + os.writeRaw(contentData_.cdata(), contentData_.size_bytes()); } else { // Other cases are less fortunate, and no std::string_view - std::string str(data.cdata(), data.size_bytes()); + std::string str(contentData_.cdata(), contentData_.size_bytes()); os.writeQuoted(str, false); } @@ -1044,29 +1016,33 @@ bool Foam::decomposedBlockData::writeObject const bool valid ) const { - // Always write BINARY - streamOpt.format(IOstream::BINARY); - autoPtr osPtr; if (UPstream::master(comm_)) { // Note: always write binary. These are strings so readable anyway. // They have already be tokenised on the sending side. - osPtr.reset(new OFstream(objectPath(), streamOpt)); - IOobject::writeHeader(*osPtr); + + osPtr.reset(new OFstream(objectPath(), IOstreamOption::BINARY)); + + decomposedBlockData::writeHeader + ( + *osPtr, + streamOpt, // streamOpt for data + static_cast(*this) + ); } labelList recvSizes; - gather(comm_, label(this->size_bytes()), recvSizes); + gather(comm_, label(contentData_.size_bytes()), recvSizes); - List start; + List blockOffsets; PtrList> slaveData; // dummy slave data return writeBlocks ( comm_, osPtr, - start, - *this, + blockOffsets, + contentData_, recvSizes, slaveData, commsType_ @@ -1074,51 +1050,4 @@ bool Foam::decomposedBlockData::writeObject } -Foam::label Foam::decomposedBlockData::numBlocks(const fileName& fName) -{ - label nBlocks = 0; - - IFstream is(fName); - is.fatalCheck(FUNCTION_NAME); - - if (!is.good()) - { - return nBlocks; - } - - // FoamFile header - token firstToken(is); - - if (is.good() && firstToken.isWord("FoamFile")) - { - dictionary headerDict(is); - is.version(headerDict.get("version")); - is.format(headerDict.get("format")); - - // Obtain number of blocks directly - if (headerDict.readIfPresent("blocks", nBlocks)) - { - return nBlocks; - } - } - - // Fallback to brute force read of each data block - List data; - while (is.good()) - { - token sizeToken(is); - if (!sizeToken.isLabel()) - { - return nBlocks; - } - is.putBack(sizeToken); - - is >> data; - nBlocks++; - } - - return nBlocks; -} - - // ************************************************************************* // diff --git a/src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.H b/src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.H index e56059b1a2..23a99162a4 100644 --- a/src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.H +++ b/src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.H @@ -28,10 +28,56 @@ Class Foam::decomposedBlockData Description - decomposedBlockData is a List with IO on the master processor only. + The decomposedBlockData comprise a \c List\ for each output + processor, typically with IO on the master processor only. + + For decomposedBlockData, we make a distinction between the container + description and the individual block contents. + + The \b FoamFile header specifies the container characteristics and thus + has \c class = \c %decomposedBlockData and normally \c format = \c binary. + This description refers to the \em entire file container, not the + individual blocks. + + Each processor block is simply a binary chunk of characters and the + first block also contains the header description for all of the blocks. + For example, +\verbatim +FoamFile +{ + version 2.0; + format binary; + arch "LSB;label=32;scalar=64"; + class decomposedBlockData; + location "constant/polyMesh"; + object points; +} + +// processor0 +NCHARS +(FoamFile +{ + version 2.0; + format ascii; + arch "LSB;label=32;scalar=64"; + class vectorField; + location "constant/polyMesh"; + object points; +} +...content... +) + +// processor1 +NCHARS +(...content...) + +... +\endverbatim + SourceFiles decomposedBlockData.C + decomposedBlockDataHeader.C \*---------------------------------------------------------------------------*/ @@ -53,9 +99,22 @@ namespace Foam class decomposedBlockData : - public regIOobject, - public List + public regIOobject { + // Private Functions + + //- Helper: write content for FoamFile IOobject header + static void writeHeaderContent + ( + Ostream& os, + IOstreamOption streamOptContainer, + const word& objectType, + const string& note, + const fileName& location, + const word& objectName + ); + + protected: // Protected Data @@ -66,11 +125,14 @@ protected: //- Communicator for all parallel comms const label comm_; + //- The block content + List contentData_; + // Protected Member Functions //- Helper: determine number of processors whose recvSizes fits - // ito maxBufferSize + //- into maxBufferSize static label calcNumProcs ( const label comm, @@ -84,7 +146,7 @@ protected: ( const label comm, autoPtr& isPtr, - List& data, + List& contentChars, const UPstream::commsTypes commsType ); @@ -105,30 +167,12 @@ public: const UPstream::commsTypes = UPstream::commsTypes::scheduled ); - //- Construct given an IOobject and for READ_IF_MODIFIED a List - decomposedBlockData - ( - const label comm, - const IOobject& io, - const UList& list, - const UPstream::commsTypes = UPstream::commsTypes::scheduled - ); - - //- Construct by transferring the List contents - decomposedBlockData - ( - const label comm, - const IOobject& io, - List&& list, - const UPstream::commsTypes = UPstream::commsTypes::scheduled - ); - //- Destructor virtual ~decomposedBlockData() = default; - // Member functions + // Member Functions //- Read object virtual bool read(); @@ -147,30 +191,71 @@ public: // Helpers - //- Read header. Call only on master. - static bool readMasterHeader(IOobject&, Istream&); + //- True if object type is a known collated type + static bool isCollatedType(const word& objectType); + + //- True if object header class is a known collated type + static bool isCollatedType(const IOobject& io); + + //- Read header as per IOobject with additional handling of + //- decomposedBlockData + static bool readHeader(IOobject& io, Istream& is); //- Helper: write FoamFile IOobject header static void writeHeader ( Ostream& os, - IOstreamOption streamOpt, + IOstreamOption streamOptContainer, const word& objectType, const string& note, const fileName& location, const word& objectName ); + //- Helper: write FoamFile IOobject header + static void writeHeader + ( + Ostream& os, + IOstreamOption streamOptData, + const IOobject& io + ); + + //- Helper: read block of (binary) character data + static bool readBlockEntry + ( + Istream& is, + List& charData + ); + + //- Helper: write block of (binary) character data + static std::streamoff writeBlockEntry + ( + OSstream& os, + const label blocki, + const UList& charData + ); + + //- Helper: write block of (binary) character data + // \return -1 on error + static std::streamoff writeBlockEntry + ( + OSstream& os, + IOstreamOption streamOptData, + const regIOobject& io, + const label blocki, + const bool withLocalHeader + ); + //- Read selected block (non-seeking) + header information static autoPtr readBlock ( const label blocki, - Istream& is, + ISstream& is, IOobject& headerIO ); //- Read master header information (into headerIO) and return - // data in stream. Note: isPtr is only valid on master. + //- data in stream. Note: isPtr is only valid on master. static autoPtr readBlocks ( const label comm, @@ -190,9 +275,11 @@ public: labelList& datas ); - //- Helper: gather data from (subset of) slaves. Returns - // recvData : received data - // recvOffsets : offset in data. recvOffsets is nProcs+1 + //- Helper: gather data from (subset of) slaves. + // + // Returns: + // - recvData : received data + // - recvOffsets : offset in data. recvOffsets is nProcs+1 static void gatherSlaveData ( const label comm, @@ -206,13 +293,14 @@ public: List& recvData ); - //- Write *this. Ostream only valid on master. Returns starts of - // processor blocks + //- Write *this. Ostream only valid on master. + // Returns offsets of processor blocks in blockOffset static bool writeBlocks ( const label comm, autoPtr& osPtr, - List& start, + List& blockOffset, + const UList& masterData, const labelUList& recvSizes, @@ -223,9 +311,6 @@ public: const UPstream::commsTypes, const bool syncReturnState = true ); - - //- Detect number of blocks in a file - static label numBlocks(const fileName& fName); }; diff --git a/src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockDataHeader.C b/src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockDataHeader.C new file mode 100644 index 0000000000..b1ccebb754 --- /dev/null +++ b/src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockDataHeader.C @@ -0,0 +1,217 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | OpenFOAM: The Open Source CFD Toolbox + \\ / O peration | + \\ / A nd | www.openfoam.com + \\/ M anipulation | +------------------------------------------------------------------------------- + Copyright (C) 2021 OpenCFD Ltd. +------------------------------------------------------------------------------- +License + This file is part of OpenFOAM. + + OpenFOAM is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with OpenFOAM. If not, see . + +\*---------------------------------------------------------------------------*/ + +#include "decomposedBlockData.H" +#include "dictionary.H" +#include "foamVersion.H" +#include "objectRegistry.H" +#include "ListStream.H" + +// * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * // + +namespace Foam +{ + +inline void writeSpaces(Ostream& os, label nSpaces) +{ + if (nSpaces < 1) + { + nSpaces = 1; + } + while (nSpaces--) + { + os.write(char(token::SPACE)); + } +} + +// Similar to writeEntry, but with fewer spaces +template +inline void writeHeaderEntry(Ostream& os, const word& key, const T& value) +{ + os << indent << key; + writeSpaces(os, 12 - label(key.size())); + os << value << char(token::END_STATEMENT) << nl; +} + +} // End namespace Foam + + +// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * // + +void Foam::decomposedBlockData::writeHeaderContent +( + Ostream& os, + IOstreamOption streamOptContainer, + const word& objectType, + const string& note, + const fileName& location, + const word& objectName +) +{ + // Standard header entries + writeHeaderEntry(os, "version", streamOptContainer.version()); + writeHeaderEntry(os, "format", streamOptContainer.format()); + writeHeaderEntry(os, "arch", foamVersion::buildArch); + + if (!note.empty()) + { + writeHeaderEntry(os, "note", note); + } + + if (objectType.empty()) + { + // Empty type not allowed - use 'dictionary' fallback + writeHeaderEntry(os, "class", word("dictionary")); + } + else + { + writeHeaderEntry(os, "class", objectType); + } + + if (!location.empty()) + { + writeHeaderEntry(os, "location", location); + } + writeHeaderEntry(os, "object", objectName); +} + + +// * * * * * * * * * * * * * * * Members Functions * * * * * * * * * * * * * // + +bool Foam::decomposedBlockData::readHeader(IOobject& io, Istream& is) +{ + dictionary headerDict; + + // Read the regular "FoamFile" header + bool ok = io.readHeader(headerDict, is); + + if (decomposedBlockData::isCollatedType(io)) + { + // Quick information - extract from "data.class" + if (headerDict.readIfPresent("data.class", io.headerClassName())) + { + return ok; + } + + { + // Master-only reading of header + List charData; + decomposedBlockData::readBlockEntry(is, charData); + + UIListStream headerStream(charData); + headerStream.name() = is.name(); + + ok = io.readHeader(headerStream); + } + } + + return ok; +} + + +void Foam::decomposedBlockData::writeHeader +( + Ostream& os, + IOstreamOption streamOptContainer, + const word& objectType, + const string& note, + const fileName& location, + const word& objectName +) +{ + if (IOobject::bannerEnabled()) + { + IOobject::writeBanner(os); + } + + os.beginBlock("FoamFile"); + + decomposedBlockData::writeHeaderContent + ( + os, + streamOptContainer, + objectType, + note, + location, + objectName + ); + + os.endBlock(); + + if (IOobject::bannerEnabled()) + { + IOobject::writeDivider(os) << nl; + } +} + + +void Foam::decomposedBlockData::writeHeader +( + Ostream& os, + IOstreamOption streamOptData, + const IOobject& io +) +{ + if (IOobject::bannerEnabled()) + { + IOobject::writeBanner(os); + } + + os.beginBlock("FoamFile"); + + decomposedBlockData::writeHeaderContent + ( + os, + static_cast(os), // streamOpt container + decomposedBlockData::typeName, // class + io.note(), + (io.instance()/io.db().dbDir()/io.local()), // location + io.name() + ); + + { + writeHeaderEntry(os, "data.format", streamOptData.format()); + writeHeaderEntry(os, "data.class", io.type()); + } + + // Meta-data (if any) + const dictionary* metaDataDict = io.findMetaData(); + if (metaDataDict && !metaDataDict->empty()) + { + metaDataDict->writeEntry("meta", os); + } + + os.endBlock(); + + if (IOobject::bannerEnabled()) + { + IOobject::writeDivider(os) << nl; + } +} + + +// ************************************************************************* // diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.C b/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.C index 3ecbd4d1ca..9f76817ed3 100644 --- a/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.C +++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.C @@ -58,6 +58,7 @@ bool Foam::OFstreamCollator::writeFile Pout<< "OFstreamCollator : Writing master " << masterData.size() << " bytes to " << fName << " using comm " << comm << endl; + if (slaveData.size()) { Pout<< "OFstreamCollator : Slave data" << endl; @@ -85,10 +86,10 @@ bool Foam::OFstreamCollator::writeFile decomposedBlockData::writeHeader ( *osPtr, - streamOpt, + streamOpt, // streamOpt for container objectType, "", // note - fName, // location + "", // location (leave empty instead inaccurate) fName.name() // object name ); } @@ -106,12 +107,12 @@ bool Foam::OFstreamCollator::writeFile // the master processor in order. However can be unstable // for some mpi so default is non-blocking. - List start; + List blockOffset; decomposedBlockData::writeBlocks ( comm, osPtr, - start, + blockOffset, slice, recvSizes, slaveData, @@ -141,11 +142,8 @@ bool Foam::OFstreamCollator::writeFile { sum += recv; } - // Use ostringstream to display long int (until writing these is - // supported) - std::ostringstream os; - os << sum; - Pout<< " (overall " << os.str() << ")"; + // Use std::to_string to display long int + Pout<< " (overall " << std::to_string(sum) << ')'; } Pout<< " to " << fName << " using comm " << comm << endl; @@ -355,10 +353,10 @@ bool Foam::OFstreamCollator::write off_t totalSize = 0; label maxLocalSize = 0; { - for (label proci = 0; proci < recvSizes.size(); proci++) + for (const label recvSize : recvSizes) { - totalSize += recvSizes[proci]; - maxLocalSize = max(maxLocalSize, recvSizes[proci]); + totalSize += recvSize; + maxLocalSize = max(maxLocalSize, recvSize); } Pstream::scatter(totalSize, Pstream::msgType(), localComm_); Pstream::scatter(maxLocalSize, Pstream::msgType(), localComm_); @@ -443,7 +441,7 @@ bool Foam::OFstreamCollator::write ( UPstream::commsTypes::nonBlocking, proci, - reinterpret_cast(slaveData[proci].data()), + slaveData[proci].data(), slaveData[proci].size_bytes(), Pstream::msgType(), localComm_ @@ -458,7 +456,7 @@ bool Foam::OFstreamCollator::write ( UPstream::commsTypes::nonBlocking, 0, - reinterpret_cast(slice.cdata()), + slice.cdata(), slice.size_bytes(), Pstream::msgType(), localComm_ diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.H b/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.H index 636a0309b7..4e2393bb98 100644 --- a/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.H +++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.H @@ -71,10 +71,8 @@ class OFstreamCollator { // Private Class - class writeData + struct writeData { - public: - const label comm_; const word objectType_; const fileName pathName_; @@ -105,18 +103,18 @@ class OFstreamCollator append_(append) {} - //- (approximate) size of master + any optional slave data + //- The (approximate) size of master + any optional slave data off_t size() const { - off_t sz = data_.size(); + off_t totalSize = data_.size(); forAll(slaveData_, i) { if (slaveData_.set(i)) { - sz += slaveData_[i].size(); + totalSize += slaveData_[i].size(); } } - return sz; + return totalSize; } }; diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C b/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C index 1fd1bd0ecd..393b46ba15 100644 --- a/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C +++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C @@ -74,13 +74,13 @@ namespace fileOperations } -// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * // +// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * // Foam::labelList Foam::fileOperations::collatedFileOperation::ioRanks() { labelList ioRanks; - string ioRanksString(getEnv("FOAM_IORANKS")); + string ioRanksString(Foam::getEnv("FOAM_IORANKS")); if (!ioRanksString.empty()) { IStringStream is(ioRanksString); @@ -91,6 +91,102 @@ Foam::labelList Foam::fileOperations::collatedFileOperation::ioRanks() } +// * * * * * * * * * * * * Protected Member Functions * * * * * * * * * * * // + +void Foam::fileOperations::collatedFileOperation::printBanner +( + const bool printRanks +) const +{ + DetailInfo + << "I/O : " << this->type(); + + if (maxThreadFileBufferSize == 0) + { + DetailInfo + << " [unthreaded] (maxThreadFileBufferSize = 0)." << nl + << " Writing may be slow for large file sizes." + << endl; + } + else + { + DetailInfo + << " [threaded] (maxThreadFileBufferSize = " + << maxThreadFileBufferSize << ")." << nl + << " Requires buffer large enough to collect all data" + " or thread support" << nl + << " enabled in MPI. If MPI thread support cannot be" + " enabled, deactivate" << nl + << " threading by setting maxThreadFileBufferSize" + " to 0 in" << nl + << " OpenFOAM etc/controlDict" << endl; + } + + if (printRanks) + { + // Information about the ranks + stringList ioRanks(Pstream::nProcs()); + if (Pstream::master(comm_)) + { + // Don't usually need the pid + // ioRanks[Pstream::myProcNo()] = hostName()+"."+name(pid()); + ioRanks[Pstream::myProcNo()] = hostName(); + } + Pstream::gatherList(ioRanks); + + DynamicList