zhyang-dev/openfoam
1
0
Fork 0
mirror of https://develop.openfoam.com/Development/openfoam.git synced 2025-11-28 03:28:01 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

ENH: additional routines for reading/writing/parsing IOObject headers

Browse Source 
- support selective enable/disable of the file banner.

ENH: improve code isolation for decomposedBlockData

- use readBlockEntry/writeBlockEntry to encapsulate the IO handling,
  which ensures more consistency

- new decomposedBlockData::readHeader for chaining into the
  block header information.

- remove unused constructors for decomposedBlockData

ENH: minor cleanup of collated fileOperations
This commit is contained in:
Mark Olesen
2021-03-05 14:14:27 +01:00
committed by Andrew Heather
parent e8cf2a2c62
commit 0c985edfc8
19 changed files with 1282 additions and 916 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
applications/test/IOField/Test-IOField.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2017 OpenFOAM Foundation Copyright (C) 2017 OpenFOAM Foundation
Copyright (C) 2019 OpenCFD Ltd. Copyright (C) 2019-2021 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
@ -96,8 +96,8 @@ void writeAndRead
Pout<< "** Writing:" << writeType Pout<< "** Writing:" << writeType
<< " Reading:" << readType << endl; << " Reading:" << readType << endl;
autoPtr<fileOperation> writeHandler(fileOperation::New(writeType, true)); // The write handler
fileHandler(writeHandler); fileHandler(fileOperation::New(writeType, true));
// Delete // Delete
Pout<< "Deleting:" << fileHandler().filePath(io.objectPath()) << endl; Pout<< "Deleting:" << fileHandler().filePath(io.objectPath()) << endl;
@ -107,8 +107,8 @@ void writeAndRead
Pout<< "Writing:" << fileHandler().objectPath(io, io.name()) << endl; Pout<< "Writing:" << fileHandler().objectPath(io, io.name()) << endl;
doWrite<Type>(io, sz); doWrite<Type>(io, sz);
autoPtr<fileOperation> readHandler(fileOperation::New(readType, true)); // The read handler
fileHandler(readHandler); fileHandler(fileOperation::New(readType, true));
// Read // Read
IOobject readIO(io); IOobject readIO(io);
@ -130,8 +130,7 @@ void readIfPresent
const word& readType const word& readType
) )
{ {
autoPtr<fileOperation> readHandler(fileOperation::New(readType, true)); fileHandler(fileOperation::New(readType, true));
fileHandler(readHandler);
// Read // Read
Pout<< "Reading:" << fileHandler().filePath(io.objectPath()) << endl; Pout<< "Reading:" << fileHandler().filePath(io.objectPath()) << endl;
@ -178,6 +177,7 @@ void doTests(IOobject& io, const label sz)
int main(int argc, char *argv[]) int main(int argc, char *argv[])
{ {
argList::noBanner();
argList::addBoolOption("bool", "Use bool for tests"); argList::addBoolOption("bool", "Use bool for tests");
argList::addBoolOption("scalar", "Use scalar for tests"); argList::addBoolOption("scalar", "Use scalar for tests");
argList::addBoolOption("label", "Use label for tests (default)"); argList::addBoolOption("label", "Use label for tests (default)");
@ -209,6 +209,11 @@ int main(int argc, char *argv[])
IOobject::NO_WRITE IOobject::NO_WRITE
); );
{
dictionary headerDict;
io.writeHeader(headerDict, "anything", IOstreamOption());
Info<< "IOobjectHeader" << headerDict << nl;
}
label tested = 0; label tested = 0;

2
src/OpenFOAM/Make/files
View File

@ -311,6 +311,7 @@ $(IOdictionary)/unwatchedIOdictionary.C
db/IOobjects/IOMap/IOMapName.C db/IOobjects/IOMap/IOMapName.C
db/IOobjects/decomposedBlockData/decomposedBlockData.C db/IOobjects/decomposedBlockData/decomposedBlockData.C
db/IOobjects/decomposedBlockData/decomposedBlockDataHeader.C
db/IOobjects/GlobalIOField/GlobalIOFields.C db/IOobjects/GlobalIOField/GlobalIOFields.C
@ -320,6 +321,7 @@ db/IOobjects/GlobalIOList/globalIOLists.C
IOobject = db/IOobject IOobject = db/IOobject
$(IOobject)/IOobject.C $(IOobject)/IOobject.C
$(IOobject)/IOobjectIO.C $(IOobject)/IOobjectIO.C
$(IOobject)/IOobjectMetaData.C
$(IOobject)/IOobjectReadHeader.C $(IOobject)/IOobjectReadHeader.C
$(IOobject)/IOobjectWriteHeader.C $(IOobject)/IOobjectWriteHeader.C

10
src/OpenFOAM/db/IOobject/IOobject.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2016-2020 OpenCFD Ltd. Copyright (C) 2016-2021 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
@ -28,7 +28,7 @@ License
#include "IOobject.H" #include "IOobject.H"
#include "Time.H" #include "Time.H"
#include "IFstream.H" #include "Istream.H"
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * // // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
@ -37,6 +37,8 @@ namespace Foam
defineTypeNameAndDebug(IOobject, 0); defineTypeNameAndDebug(IOobject, 0);
} }
bool Foam::IOobject::bannerEnabled_(true);
char Foam::IOobject::scopeSeparator char Foam::IOobject::scopeSeparator
( (
#ifdef _WIN32 #ifdef _WIN32
@ -47,7 +49,6 @@ char Foam::IOobject::scopeSeparator
#endif #endif
); );
const Foam::Enum const Foam::Enum
< <
Foam::IOobject::fileCheckTypes Foam::IOobject::fileCheckTypes
@ -60,7 +61,6 @@ Foam::IOobject::fileCheckTypesNames
{ fileCheckTypes::inotifyMaster, "inotifyMaster" }, { fileCheckTypes::inotifyMaster, "inotifyMaster" },
}); });
// Default fileCheck type // Default fileCheck type
Foam::IOobject::fileCheckTypes Foam::IOobject::fileModificationChecking Foam::IOobject::fileCheckTypes Foam::IOobject::fileModificationChecking
( (
@ -460,7 +460,7 @@ Foam::IOobject::IOobject
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // // * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
const Foam::objectRegistry& Foam::IOobject::db() const const Foam::objectRegistry& Foam::IOobject::db() const noexcept
{ {
return db_; return db_;
} }

136
src/OpenFOAM/db/IOobject/IOobject.H
View File

@ -64,6 +64,21 @@ Description
- \par NO_WRITE - \par NO_WRITE
No automatic write on destruction but can be written explicitly No automatic write on destruction but can be written explicitly
When serializing, the IOobject characteristics are typically written
as a \c FoamFile header, which is a sub-dictionary with the following
type of content:
\table
Property | Description | Type | Reqd | Deflt
version | The base format version | float | no | 2.0
format | The stream format (ascii/binary) | word | yes |
arch | The architecture string | string | no |
note | Descriptive note about the object | string | no |
location | The relative location of the object | string | no |
class | The type of the object | word | yes |
object | The name of the object | word | yes |
\endtable
Note Note
Specifying registered does not result in the IOobject itself being Specifying registered does not result in the IOobject itself being
registered. It is only serves as guidance for a regIOobject using it. registered. It is only serves as guidance for a regIOobject using it.
@ -86,6 +101,7 @@ SourceFiles
#include "fileName.H" #include "fileName.H"
#include "typeInfo.H" #include "typeInfo.H"
#include "autoPtr.H" #include "autoPtr.H"
#include "IOstreamOption.H"
#include "InfoProxy.H" #include "InfoProxy.H"
#include "Enum.H" #include "Enum.H"
@ -96,6 +112,7 @@ namespace Foam
// Forward Declarations // Forward Declarations
class Time; class Time;
class dictionary;
class objectRegistry; class objectRegistry;
/*---------------------------------------------------------------------------*\ /*---------------------------------------------------------------------------*\
@ -146,6 +163,12 @@ public:
private: private:
// Static Data Members
//- Use an output file banner, enabled by default
static bool bannerEnabled_;
// Private Data // Private Data
//- Name //- Name
@ -192,6 +215,27 @@ protected:
// Protected Member Functions // Protected Member Functions
//- Helper: write content for FoamFile IOobject header
//- with optional meta information.
static void writeHeaderContent
(
Ostream& os,
const IOobject& io,
const word& objectType,
const dictionary* metaDataDict = nullptr
);
//- Helper: write dictionary content for FoamFile header
//- with optional meta information.
static void writeHeaderContent
(
dictionary& os,
const IOobject& io,
const word& objectType,
IOstreamOption streamOpt,
const dictionary* metaDataDict = nullptr
);
//- Set the object state to bad //- Set the object state to bad
void setBad(const string& s); void setBad(const string& s);
@ -214,6 +258,21 @@ public:
// Static Functions // Static Functions
//- Status of output file banner
static bool bannerEnabled() noexcept
{
return bannerEnabled_;
}
//- Enable/disable an output file banner
// \return the previous value
static bool bannerEnabled(bool on) noexcept
{
bool old(bannerEnabled_);
bannerEnabled_ = on;
return old;
}
//- Split path into instance, local, name components //- Split path into instance, local, name components
// //
// The splitting behaviour is as follows: // The splitting behaviour is as follows:
@ -364,28 +423,28 @@ public:
// Member Functions // Member Functions
// General access // General Access
//- Return the local objectRegistry //- Return the local objectRegistry
const objectRegistry& db() const; const objectRegistry& db() const noexcept;
//- Return Time associated with the objectRegistry //- Return Time associated with the objectRegistry
const Time& time() const; const Time& time() const;
//- Return name //- Return name
inline const word& name() const; inline const word& name() const noexcept;
//- Return name of the class name read from header //- Return name of the class name read from header
inline const word& headerClassName() const; inline const word& headerClassName() const noexcept;
//- Return non-constant access to the class name read from header //- Return non-constant access to the class name read from header
inline word& headerClassName(); inline word& headerClassName() noexcept;
//- Return the optional note //- Return the optional note
inline const string& note() const; inline const string& note() const noexcept;
//- Return non-constant access to the optional note //- Return non-constant access to the optional note
inline string& note(); inline string& note() noexcept;
//- Rename //- Rename
virtual void rename(const word& newName) virtual void rename(const word& newName)
@ -394,16 +453,16 @@ public:
} }
//- Should object created with this IOobject be registered? //- Should object created with this IOobject be registered?
inline bool registerObject() const; inline bool registerObject() const noexcept;
//- Should object created with this IOobject be registered? //- Should object created with this IOobject be registered?
inline bool& registerObject(); inline bool& registerObject() noexcept;
//- Is object same for all processors? //- Is object same for all processors?
inline bool globalObject() const; inline bool globalObject() const noexcept;
//- Is object same for all processors? //- Is object same for all processors?
inline bool& globalObject(); inline bool& globalObject() noexcept;
//- The sizeof (label) in bytes, possibly read from the header //- The sizeof (label) in bytes, possibly read from the header
inline unsigned labelByteSize() const noexcept; inline unsigned labelByteSize() const noexcept;
@ -422,19 +481,25 @@ public:
inline bool isHeaderClassName() const; inline bool isHeaderClassName() const;
// Meta-data
//- Return pointer to meta-data (if any) or nullptr
virtual const dictionary* findMetaData() const noexcept;
// Read/write options // Read/write options
//- The read option //- The read option
inline readOption readOpt() const; inline readOption readOpt() const noexcept;
//- Non-constant access to the read option //- Non-constant access to the read option
inline readOption& readOpt(); inline readOption& readOpt() noexcept;
//- The write option //- The write option
inline writeOption writeOpt() const; inline writeOption writeOpt() const noexcept;
//- Non-constant access to the write option //- Non-constant access to the write option
inline writeOption& writeOpt(); inline writeOption& writeOpt() noexcept;
// Path components // Path components
@ -449,11 +514,11 @@ public:
const fileName& caseName() const; const fileName& caseName() const;
inline const fileName& instance() const; inline const fileName& instance() const noexcept;
inline fileName& instance(); inline fileName& instance() noexcept;
inline const fileName& local() const; inline const fileName& local() const noexcept;
//- The complete path //- The complete path
fileName path() const; fileName path() const;
@ -489,9 +554,19 @@ public:
// Reading // Reading
//- Read header //- Parse 'FoamFile' header contents and set the IOobject
//- characteristics and return the stream characteristics.
IOstreamOption parseHeader(const dictionary& headerDict);
//- Read header ('FoamFile' dictionary) and set the
//- IOobject and stream characteristics.
bool readHeader(Istream& is); bool readHeader(Istream& is);
//- Read header (the 'FoamFile' dictionary) and set the
//- IOobject and stream characteristics.
// Saves the header content in the given dictionary.
bool readHeader(dictionary& headerDict, Istream& is);
//- Read header (uses typeFilePath to find file) and check its info. //- Read header (uses typeFilePath to find file) and check its info.
// Optionally checks headerClassName against the type-name. // Optionally checks headerClassName against the type-name.
// When search is false, simply use the current instance, // When search is false, simply use the current instance,
@ -521,24 +596,31 @@ public:
//- Write the standard end file divider //- Write the standard end file divider
static Ostream& writeEndDivider(Ostream& os); static Ostream& writeEndDivider(Ostream& os);
//- Write header with current type() and arch information //- Write header with current type()
bool writeHeader(Ostream& os) const; bool writeHeader(Ostream& os) const;
//- Write header with override of type and optionally without //- Write header with override of type
//- arch information in ASCII bool writeHeader(Ostream& os, const word& objectType) const;
bool writeHeader
//- Write header into a dictionary with current type()
//- and given output format
void writeHeader(dictionary& dict, IOstreamOption streamOpt) const;
//- Write header into a dictionary with override of type
//- and given output format
void writeHeader
( (
Ostream& os, dictionary& dict,
const word& objectType, const word& objectType,
const bool noArchAscii = false IOstreamOption streamOpt
) const; ) const;
// Error Handling // Error Handling
inline bool good() const; inline bool good() const noexcept;
inline bool bad() const; inline bool bad() const noexcept;
// Info // Info

36
src/OpenFOAM/db/IOobject/IOobjectI.H
View File

@ -67,7 +67,7 @@ inline Foam::word Foam::IOobject::scopedName
// General access // General access
inline const Foam::word& Foam::IOobject::name() const inline const Foam::word& Foam::IOobject::name() const noexcept
{ {
return name_; return name_;
} }
@ -85,49 +85,49 @@ inline Foam::word Foam::IOobject::member() const
} }
inline const Foam::word& Foam::IOobject::headerClassName() const inline const Foam::word& Foam::IOobject::headerClassName() const noexcept
{ {
return headerClassName_; return headerClassName_;
} }
inline Foam::word& Foam::IOobject::headerClassName() inline Foam::word& Foam::IOobject::headerClassName() noexcept
{ {
return headerClassName_; return headerClassName_;
} }
inline const Foam::string& Foam::IOobject::note() const inline const Foam::string& Foam::IOobject::note() const noexcept
{ {
return note_; return note_;
} }
inline Foam::string& Foam::IOobject::note() inline Foam::string& Foam::IOobject::note() noexcept
{ {
return note_; return note_;
} }
inline bool Foam::IOobject::registerObject() const inline bool Foam::IOobject::registerObject() const noexcept
{ {
return registerObject_; return registerObject_;
} }
inline bool& Foam::IOobject::registerObject() inline bool& Foam::IOobject::registerObject() noexcept
{ {
return registerObject_; return registerObject_;
} }
inline bool Foam::IOobject::globalObject() const inline bool Foam::IOobject::globalObject() const noexcept
{ {
return globalObject_; return globalObject_;
} }
inline bool& Foam::IOobject::globalObject() inline bool& Foam::IOobject::globalObject() noexcept
{ {
return globalObject_; return globalObject_;
} }
@ -162,25 +162,25 @@ inline bool Foam::IOobject::isHeaderClassName() const
// Read/write options // Read/write options
inline Foam::IOobject::readOption Foam::IOobject::readOpt() const inline Foam::IOobject::readOption Foam::IOobject::readOpt() const noexcept
{ {
return rOpt_; return rOpt_;
} }
inline Foam::IOobject::readOption& Foam::IOobject::readOpt() inline Foam::IOobject::readOption& Foam::IOobject::readOpt() noexcept
{ {
return rOpt_; return rOpt_;
} }
inline Foam::IOobject::writeOption Foam::IOobject::writeOpt() const inline Foam::IOobject::writeOption Foam::IOobject::writeOpt() const noexcept
{ {
return wOpt_; return wOpt_;
} }
inline Foam::IOobject::writeOption& Foam::IOobject::writeOpt() inline Foam::IOobject::writeOption& Foam::IOobject::writeOpt() noexcept
{ {
return wOpt_; return wOpt_;
} }
@ -188,19 +188,19 @@ inline Foam::IOobject::writeOption& Foam::IOobject::writeOpt()
// Path components // Path components
inline const Foam::fileName& Foam::IOobject::instance() const inline const Foam::fileName& Foam::IOobject::instance() const noexcept
{ {
return instance_; return instance_;
} }
inline Foam::fileName& Foam::IOobject::instance() inline Foam::fileName& Foam::IOobject::instance() noexcept
{ {
return instance_; return instance_;
} }
inline const Foam::fileName& Foam::IOobject::local() const inline const Foam::fileName& Foam::IOobject::local() const noexcept
{ {
return local_; return local_;
} }
@ -214,13 +214,13 @@ inline Foam::fileName Foam::IOobject::objectPath() const
// Error Handling // Error Handling
inline bool Foam::IOobject::good() const inline bool Foam::IOobject::good() const noexcept
{ {
return objState_ == GOOD; return objState_ == GOOD;
} }
inline bool Foam::IOobject::bad() const inline bool Foam::IOobject::bad() const noexcept
{ {
return objState_ == BAD; return objState_ == BAD;
} }

39
src/OpenFOAM/db/IOobject/IOobjectMetaData.C Normal file
View File

@ -0,0 +1,39 @@
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2021 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
\*---------------------------------------------------------------------------*/
#include "IOobject.H"
#include "dictionary.H"
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
const Foam::dictionary* Foam::IOobject::findMetaData() const noexcept
{
return nullptr;
}
// ************************************************************************* //

88
src/OpenFOAM/db/IOobject/IOobjectReadHeader.C
View File

@ -32,7 +32,45 @@ License
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // // * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
bool Foam::IOobject::readHeader(Istream& is) Foam::IOstreamOption Foam::IOobject::parseHeader(const dictionary& headerDict)
{
IOstreamOption streamOpt; // == (ASCII, currentVersion)
// Treat "version" as optional
{
token tok;
if (headerDict.readIfPresent("version", tok))
{
streamOpt.version(tok);
}
}
// Treat "format" as mandatory, could also as optional
streamOpt.format(headerDict.get<word>("format"));
headerClassName_ = headerDict.get<word>("class");
const word headerObject(headerDict.get<word>("object"));
// The "note" entry is optional
headerDict.readIfPresent("note", note_);
// The "arch" information may be missing
string arch;
if (headerDict.readIfPresent("arch", arch))
{
unsigned val = foamVersion::labelByteSize(arch);
if (val) sizeofLabel_ = static_cast<unsigned char>(val);
val = foamVersion::scalarByteSize(arch);
if (val) sizeofScalar_ = static_cast<unsigned char>(val);
}
return streamOpt;
}
bool Foam::IOobject::readHeader(dictionary& headerDict, Istream& is)
{ {
if (IOobject::debug) if (IOobject::debug)
{ {
@ -64,48 +102,21 @@ bool Foam::IOobject::readHeader(Istream& is)
if (is.good() && firstToken.isWord("FoamFile")) if (is.good() && firstToken.isWord("FoamFile"))
{ {
const dictionary headerDict(is); headerDict.read(is, false); // Read sub-dictionary content
is.version(headerDict.get<token>("version")); IOstreamOption streamOpt = parseHeader(headerDict);
is.format(headerDict.get<word>("format"));
headerClassName_ = headerDict.get<word>("class");
const word headerObject(headerDict.get<word>("object"));
if (IOobject::debug && headerObject != name())
{
IOWarningInFunction(is)
<< " object renamed from "
<< name() << " to " << headerObject
<< " for file " << is.name() << endl;
}
// The note entry is optional
headerDict.readIfPresent("note", note_);
sizeofLabel_ = sizeof(label);
sizeofScalar_ = sizeof(scalar);
// The arch information is optional
string arch;
if (headerDict.readIfPresent("arch", arch))
{
unsigned val = foamVersion::labelByteSize(arch);
if (val) sizeofLabel_ = static_cast<unsigned char>(val);
val = foamVersion::scalarByteSize(arch);
if (val) sizeofScalar_ = static_cast<unsigned char>(val);
}
is.format(streamOpt.format());
is.version(streamOpt.version());
is.setLabelByteSize(sizeofLabel_); is.setLabelByteSize(sizeofLabel_);
is.setScalarByteSize(sizeofScalar_); is.setScalarByteSize(sizeofScalar_);
} }
else else
{ {
IOWarningInFunction(is) IOWarningInFunction(is)
<< "First token could not be read or is not the keyword 'FoamFile'" << "First token could not be read or is not 'FoamFile'"
<< nl << nl << "Check header is of the form:" << nl << endl; << nl << nl
<< "Check header is of the form:" << nl << endl;
writeHeader(Info); writeHeader(Info);
@ -151,4 +162,11 @@ bool Foam::IOobject::readHeader(Istream& is)
} }
bool Foam::IOobject::readHeader(Istream& is)
{
dictionary headerDict;
return IOobject::readHeader(headerDict, is);
}
// ************************************************************************* // // ************************************************************************* //

193
src/OpenFOAM/db/IOobject/IOobjectWriteHeader.C
View File

@ -27,10 +27,40 @@ License
\*---------------------------------------------------------------------------*/ \*---------------------------------------------------------------------------*/
#include "IOobject.H" #include "IOobject.H"
#include "dictionary.H"
#include "objectRegistry.H" #include "objectRegistry.H"
#include "foamVersion.H" #include "foamVersion.H"
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // // * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
namespace Foam
{
inline void writeSpaces(Ostream& os, label nSpaces)
{
if (nSpaces < 1)
{
nSpaces = 1;
}
while (nSpaces--)
{
os.write(char(token::SPACE));
}
}
// Similar to writeEntry, but with fewer spaces
template<class T>
inline void writeHeaderEntry(Ostream& os, const word& key, const T& value)
{
os << indent << key;
writeSpaces(os, 12 - label(key.size()));
os << value << char(token::END_STATEMENT) << nl;
}
} // End namespace Foam
// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * //
// A banner corresponding to this: // A banner corresponding to this:
// //
@ -117,11 +147,93 @@ Foam::Ostream& Foam::IOobject::writeEndDivider(Ostream& os)
} }
// * * * * * * * * * * * * Protected Member Functions * * * * * * * * * * * //
void Foam::IOobject::writeHeaderContent
(
Ostream& os,
const IOobject& io,
const word& objectType,
const dictionary* metaDataDict
)
{
// Standard header entries
writeHeaderEntry(os, "version", os.version());
writeHeaderEntry(os, "format", os.format());
writeHeaderEntry(os, "arch", foamVersion::buildArch);
if (!io.note().empty())
{
writeHeaderEntry(os, "note", io.note());
}
if (objectType.empty())
{
// Empty type not allowed - use 'dictionary' fallback
writeHeaderEntry(os, "class", word("dictionary"));
}
else
{
writeHeaderEntry(os, "class", objectType);
}
writeHeaderEntry(os, "location", io.instance()/io.db().dbDir()/io.local());
writeHeaderEntry(os, "object", io.name());
// Meta-data (if any)
if (metaDataDict && !metaDataDict->empty())
{
metaDataDict->writeEntry("meta", os);
}
}
void Foam::IOobject::writeHeaderContent
(
dictionary& dict,
const IOobject& io,
const word& objectType,
IOstreamOption streamOpt,
const dictionary* metaDataDict
)
{
// Standard header entries
dict.set("version", streamOpt.version());
dict.set("format", streamOpt.format());
dict.set("arch", foamVersion::buildArch);
if (!io.note().empty())
{
dict.set("note", io.note());
}
if (objectType.empty())
{
// Empty type not allowed - use 'dictionary' fallback
dict.set("class", word("dictionary"));
}
else
{
dict.set("class", objectType);
}
dict.set("location", io.instance()/io.db().dbDir()/io.local());
dict.set("object", io.name());
// Deep-copy of meta-data (if any)
if (metaDataDict && !metaDataDict->empty())
{
dict.add("meta", *metaDataDict);
}
}
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
bool Foam::IOobject::writeHeader bool Foam::IOobject::writeHeader
( (
Ostream& os, Ostream& os,
const word& objectType, const word& objectType
const bool noArchAscii
) const ) const
{ {
if (!os.good()) if (!os.good())
@ -133,39 +245,28 @@ bool Foam::IOobject::writeHeader
return false; return false;
} }
IOobject::writeBanner(os) if (IOobject::bannerEnabled())
<< "FoamFile" << nl
<< '{' << nl
<< " version " << os.version() << ';' << nl
<< " format " << os.format() << ';' << nl;
if (os.format() == IOstream::BINARY || !noArchAscii)
{ {
// Arch information (BINARY: always, ASCII: can disable) IOobject::writeBanner(os);
os << " arch " << foamVersion::buildArch << ';' << nl;
}
if (!note().empty())
{
os << " note " << note() << ';' << nl;
} }
os << " class "; os.beginBlock("FoamFile");
if (objectType.empty())
{
// Empty type not allowed - use 'dictionary' fallback
os << "dictionary";
}
else
{
os << objectType;
}
os << ';' << nl;
os << " location " << instance()/db().dbDir()/local() << ';' << nl // Standard header entries
<< " object " << name() << ';' << nl IOobject::writeHeaderContent
<< '}' << nl; (
os,
*this,
objectType,
this->findMetaData()
);
writeDivider(os) << nl; os.endBlock();
if (IOobject::bannerEnabled())
{
IOobject::writeDivider(os) << nl;
}
return true; return true;
} }
@ -173,7 +274,35 @@ bool Foam::IOobject::writeHeader
bool Foam::IOobject::writeHeader(Ostream& os) const bool Foam::IOobject::writeHeader(Ostream& os) const
{ {
return writeHeader(os, type()); return IOobject::writeHeader(os, this->type());
}
void Foam::IOobject::writeHeader
(
dictionary& dict,
const word& objectType,
IOstreamOption streamOpt
) const
{
IOobject::writeHeaderContent
(
dict,
*this,
objectType,
streamOpt,
this->findMetaData()
);
}
void Foam::IOobject::writeHeader
(
dictionary& dict,
IOstreamOption streamOpt
) const
{
IOobject::writeHeader(dict, this->type(), streamOpt);
} }

681
src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.C
View File

@ -34,10 +34,11 @@ License
#include "dictionary.H" #include "dictionary.H"
#include "objectRegistry.H" #include "objectRegistry.H"
#include "SubList.H" #include "SubList.H"
#include "charList.H"
#include "labelPair.H" #include "labelPair.H"
#include "masterUncollatedFileOperation.H" #include "masterUncollatedFileOperation.H"
#include "IListStream.H" #include "ListStream.H"
#include "foamVersion.H" #include "StringStream.H"
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * // // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
@ -46,6 +47,30 @@ namespace Foam
defineTypeNameAndDebug(decomposedBlockData, 0); defineTypeNameAndDebug(decomposedBlockData, 0);
} }
// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * //
bool Foam::decomposedBlockData::isCollatedType
(
const word& objectType
)
{
return
(
objectType == decomposedBlockData::typeName
);
}
bool Foam::decomposedBlockData::isCollatedType
(
const IOobject& io
)
{
return decomposedBlockData::isCollatedType(io.headerClassName());
}
// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * // // * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
Foam::decomposedBlockData::decomposedBlockData Foam::decomposedBlockData::decomposedBlockData
@ -57,7 +82,8 @@ Foam::decomposedBlockData::decomposedBlockData
: :
regIOobject(io), regIOobject(io),
commsType_(commsType), commsType_(commsType),
comm_(comm) comm_(comm),
contentData_()
{ {
// Temporary warning // Temporary warning
if (io.readOpt() == IOobject::MUST_READ_IF_MODIFIED) if (io.readOpt() == IOobject::MUST_READ_IF_MODIFIED)
@ -82,161 +108,125 @@ Foam::decomposedBlockData::decomposedBlockData
} }
Foam::decomposedBlockData::decomposedBlockData
(
const label comm,
const IOobject& io,
const UList<char>& list,
const UPstream::commsTypes commsType
)
:
regIOobject(io),
commsType_(commsType),
comm_(comm)
{
// Temporary warning
if (io.readOpt() == IOobject::MUST_READ_IF_MODIFIED)
{
WarningInFunction
<< "decomposedBlockData " << name()
<< " constructed with IOobject::MUST_READ_IF_MODIFIED"
" but decomposedBlockData does not support automatic rereading."
<< endl;
}
if
(
(
io.readOpt() == IOobject::MUST_READ
|| io.readOpt() == IOobject::MUST_READ_IF_MODIFIED
)
|| (io.readOpt() == IOobject::READ_IF_PRESENT && headerOk())
)
{
read();
}
else
{
List<char>::operator=(list);
}
}
Foam::decomposedBlockData::decomposedBlockData
(
const label comm,
const IOobject& io,
List<char>&& list,
const UPstream::commsTypes commsType
)
:
regIOobject(io),
commsType_(commsType),
comm_(comm)
{
// Temporary warning
if (io.readOpt() == IOobject::MUST_READ_IF_MODIFIED)
{
WarningInFunction
<< "decomposedBlockData " << name()
<< " constructed with IOobject::MUST_READ_IF_MODIFIED"
" but decomposedBlockData does not support automatic rereading."
<< endl;
}
List<char>::transfer(list);
if
(
(
io.readOpt() == IOobject::MUST_READ
|| io.readOpt() == IOobject::MUST_READ_IF_MODIFIED
)
|| (io.readOpt() == IOobject::READ_IF_PRESENT && headerOk())
)
{
read();
}
}
// * * * * * * * * * * * * * * * Members Functions * * * * * * * * * * * * * // // * * * * * * * * * * * * * * * Members Functions * * * * * * * * * * * * * //
bool Foam::decomposedBlockData::readMasterHeader(IOobject& io, Istream& is) bool Foam::decomposedBlockData::readBlockEntry
{
if (debug)
{
Pout<< "decomposedBlockData::readMasterHeader:"
<< " stream:" << is.name() << endl;
}
// Master-only reading of header
is.fatalCheck("read(Istream&)");
List<char> data(is);
is.fatalCheck("read(Istream&) : reading entry");
UIListStream headerStream(data);
headerStream.name() = is.name();
return io.readHeader(headerStream);
}
void Foam::decomposedBlockData::writeHeader
( (
Ostream& os, Istream& is,
IOstreamOption streamOpt, List<char>& charData
const word& objectType,
const string& note,
const fileName& location,
const word& objectName
) )
{ {
IOobject::writeBanner(os) // Handle any of these:
<< "FoamFile" << nl
<< '{' << nl
<< " version " << streamOpt.version() << ';' << nl
<< " format " << streamOpt.format() << ';' << nl
<< " arch " << foamVersion::buildArch << ';' << nl;
if (Pstream::parRun()) // 0. NCHARS (...)
// 1. List<char> NCHARS (...)
// 2. processorN List<char> NCHARS (...) ;
is.fatalCheck(FUNCTION_NAME);
token tok(is);
is.fatalCheck(FUNCTION_NAME);
// Dictionary format has primitiveEntry keyword:
const bool isDictFormat = (tok.isWord() && !tok.isCompound());
if (!isDictFormat && tok.good())
{ {
os << " blocks " << Pstream::nProcs() << ';' << nl; is.putBack(tok);
} }
if (!note.empty()) charData.readList(is);
if (isDictFormat)
{ {
os << " note " << note << ';' << nl; is.fatalCheck(FUNCTION_NAME);
is >> tok;
is.fatalCheck(FUNCTION_NAME);
// Swallow trailing ';'
if (tok.good() && !tok.isPunctuation(token::END_STATEMENT))
{
is.putBack(tok);
}
} }
os << " class "; return true;
if (objectType.empty())
{
// Empty type not allowed - use 'dictionary' fallback
os << "dictionary";
}
else
{
os << objectType;
}
os << ';' << nl;
if (!location.empty())
{
os << " location " << location << ';' << nl;
}
os << " object " << objectName << ';' << nl
<< '}' << nl;
IOobject::writeDivider(os) << nl;
} }
Foam::autoPtr<Foam::ISstream> Foam::decomposedBlockData::readBlock std::streamoff Foam::decomposedBlockData::writeBlockEntry
(
OSstream& os,
const label blocki,
const UList<char>& charData
)
{
// Offset to the beginning of this output
std::streamoff blockOffset = os.stdStream().tellp();
const word procName("processor" + Foam::name(blocki));
{
os << nl << "// " << procName << nl;
charData.writeList(os) << nl;
}
return blockOffset;
}
std::streamoff Foam::decomposedBlockData::writeBlockEntry
(
OSstream& os,
IOstreamOption streamOptData,
const regIOobject& io,
const label blocki,
const bool withLocalHeader
)
{
// String(s) from all data to write
string contentChars;
{
OStringStream os(streamOptData);
bool ok = true;
// Generate FoamFile header on master, without comment banner
if (withLocalHeader)
{
const bool old = IOobject::bannerEnabled(false);
ok = io.writeHeader(os);
IOobject::bannerEnabled(old);
}
// Write the data to the Ostream
ok = ok && io.writeData(os);
if (!ok)
{
return std::streamoff(-1);
}
contentChars = os.str();
}
// The character data
UList<char> charData
(
const_cast<char*>(contentChars.data()),
label(contentChars.size())
);
return decomposedBlockData::writeBlockEntry(os, blocki, charData);
}
Foam::autoPtr<Foam::ISstream>
Foam::decomposedBlockData::readBlock
( (
const label blocki, const label blocki,
Istream& is, ISstream& is,
IOobject& headerIO IOobject& headerIO
) )
{ {
@ -247,66 +237,64 @@ Foam::autoPtr<Foam::ISstream> Foam::decomposedBlockData::readBlock
<< endl; << endl;
} }
is.fatalCheck("read(Istream&)"); // Extracted header information
IOstreamOption streamOptData;
unsigned labelWidth = is.labelByteSize();
unsigned scalarWidth = is.scalarByteSize();
autoPtr<ISstream> realIsPtr; autoPtr<ISstream> realIsPtr;
// Read master for header
List<char> data;
decomposedBlockData::readBlockEntry(is, data);
if (blocki == 0) if (blocki == 0)
{ {
List<char> data(is);
is.fatalCheck("read(Istream&) : reading entry");
realIsPtr.reset(new IListStream(std::move(data))); realIsPtr.reset(new IListStream(std::move(data)));
realIsPtr->name() = is.name(); realIsPtr->name() = is.name();
// Read header
if (!headerIO.readHeader(*realIsPtr))
{ {
FatalIOErrorInFunction(*realIsPtr) // Read header from first block,
<< "problem while reading header for object " // advancing the stream position
<< is.name() << exit(FatalIOError); if (!headerIO.readHeader(*realIsPtr))
{
FatalIOErrorInFunction(*realIsPtr)
<< "Problem while reading header for object "
<< is.name() << nl
<< exit(FatalIOError);
}
} }
} }
else else
{ {
// Read master for header
List<char> data(is);
is.fatalCheck("read(Istream&) : reading entry");
IOstreamOption::versionNumber ver(IOstreamOption::currentVersion);
IOstreamOption::streamFormat fmt;
unsigned labelByteSize;
unsigned scalarByteSize;
{ {
// Read header from first block
UIListStream headerStream(data); UIListStream headerStream(data);
// Read header
if (!headerIO.readHeader(headerStream)) if (!headerIO.readHeader(headerStream))
{ {
FatalIOErrorInFunction(headerStream) FatalIOErrorInFunction(headerStream)
<< "problem while reading header for object " << "Problem while reading header for object "
<< is.name() << exit(FatalIOError); << is.name() << nl
<< exit(FatalIOError);
} }
ver = headerStream.version(); streamOptData = static_cast<IOstreamOption>(headerStream);
fmt = headerStream.format(); labelWidth = headerStream.labelByteSize();
labelByteSize = headerStream.labelByteSize(); scalarWidth = headerStream.scalarByteSize();
scalarByteSize = headerStream.scalarByteSize();
} }
for (label i = 1; i < blocki+1; i++) for (label i = 1; i < blocki+1; i++)
{ {
// Read and discard data, only retain the last one // Read and discard data, only retain the last one
is >> data; decomposedBlockData::readBlockEntry(is, data);
is.fatalCheck("read(Istream&) : reading entry");
} }
realIsPtr.reset(new IListStream(std::move(data))); realIsPtr.reset(new IListStream(std::move(data)));
realIsPtr->name() = is.name(); realIsPtr->name() = is.name();
// Apply master stream settings to realIsPtr // Apply stream settings
realIsPtr().format(fmt); realIsPtr().format(streamOptData.format());
realIsPtr().version(ver); realIsPtr().version(streamOptData.version());
realIsPtr().setLabelByteSize(labelByteSize); realIsPtr().setLabelByteSize(labelWidth);
realIsPtr().setScalarByteSize(scalarByteSize); realIsPtr().setScalarByteSize(scalarWidth);
} }
return realIsPtr; return realIsPtr;
@ -331,24 +319,28 @@ bool Foam::decomposedBlockData::readBlocks
bool ok = false; bool ok = false;
if (UPstream::master(comm))
{
auto& is = *isPtr;
is.fatalCheck(FUNCTION_NAME);
// Read master data
decomposedBlockData::readBlockEntry(is, data);
}
if (commsType == UPstream::commsTypes::scheduled) if (commsType == UPstream::commsTypes::scheduled)
{ {
if (UPstream::master(comm)) if (UPstream::master(comm))
{ {
Istream& is = *isPtr; // Master data already read ...
is.fatalCheck("read(Istream&)"); auto& is = *isPtr;
is.fatalCheck(FUNCTION_NAME);
// Read master data // Read and transmit slave data
{
is >> data;
is.fatalCheck("read(Istream&) : reading entry");
}
// Read slave data
for (const int proci : UPstream::subProcs(comm)) for (const int proci : UPstream::subProcs(comm))
{ {
List<char> elems(is); List<char> elems;
is.fatalCheck("read(Istream&) : reading entry"); decomposedBlockData::readBlockEntry(is, elems);
OPstream os OPstream os
( (
@ -387,20 +379,15 @@ bool Foam::decomposedBlockData::readBlocks
if (UPstream::master(comm)) if (UPstream::master(comm))
{ {
Istream& is = *isPtr; // Master data already read ...
is.fatalCheck("read(Istream&)"); auto& is = *isPtr;
is.fatalCheck(FUNCTION_NAME);
// Read master data // Read and transmit slave data
{
is >> data;
is.fatalCheck("read(Istream&) : reading entry");
}
// Read slave data
for (const int proci : UPstream::subProcs(comm)) for (const int proci : UPstream::subProcs(comm))
{ {
List<char> elems(is); List<char> elems;
is.fatalCheck("read(Istream&) : reading entry"); decomposedBlockData::readBlockEntry(is, elems);
UOPstream os(proci, pBufs); UOPstream os(proci, pBufs);
os << elems; os << elems;
@ -440,39 +427,45 @@ Foam::autoPtr<Foam::ISstream> Foam::decomposedBlockData::readBlocks
} }
bool ok = false; bool ok = false;
List<char> data; List<char> data;
autoPtr<ISstream> realIsPtr; autoPtr<ISstream> realIsPtr;
if (UPstream::master(comm))
{
auto& is = *isPtr;
is.fatalCheck(FUNCTION_NAME);
// Read master data
decomposedBlockData::readBlockEntry(is, data);
realIsPtr.reset(new IListStream(std::move(data)));
realIsPtr->name() = fName;
{
// Read header from first block,
// advancing the stream position
if (!headerIO.readHeader(*realIsPtr))
{
FatalIOErrorInFunction(*realIsPtr)
<< "Problem while reading header for object "
<< is.name() << nl
<< exit(FatalIOError);
}
}
}
if (commsType == UPstream::commsTypes::scheduled) if (commsType == UPstream::commsTypes::scheduled)
{ {
if (UPstream::master(comm)) if (UPstream::master(comm))
{ {
Istream& is = *isPtr; // Master data already read ...
is.fatalCheck("read(Istream&)"); auto& is = *isPtr;
is.fatalCheck(FUNCTION_NAME);
// Read master data // Read and transmit slave data
{
is >> data;
is.fatalCheck("read(Istream&) : reading entry");
realIsPtr.reset(new IListStream(std::move(data)));
realIsPtr->name() = fName;
// Read header
if (!headerIO.readHeader(*realIsPtr))
{
FatalIOErrorInFunction(*realIsPtr)
<< "problem while reading header for object "
<< is.name() << exit(FatalIOError);
}
}
// Read slave data
for (const int proci : UPstream::subProcs(comm)) for (const int proci : UPstream::subProcs(comm))
{ {
is >> data; decomposedBlockData::readBlockEntry(is, data);
is.fatalCheck("read(Istream&) : reading entry");
OPstream os OPstream os
( (
@ -514,31 +507,15 @@ Foam::autoPtr<Foam::ISstream> Foam::decomposedBlockData::readBlocks
if (UPstream::master(comm)) if (UPstream::master(comm))
{ {
Istream& is = *isPtr; // Master data already read ...
is.fatalCheck("read(Istream&)"); auto& is = *isPtr;
is.fatalCheck(FUNCTION_NAME);
// Read master data // Read and transmit slave data
{
is >> data;
is.fatalCheck("read(Istream&) : reading entry");
realIsPtr.reset(new IListStream(std::move(data)));
realIsPtr->name() = fName;
// Read header
if (!headerIO.readHeader(*realIsPtr))
{
FatalIOErrorInFunction(*realIsPtr)
<< "problem while reading header for object "
<< is.name() << exit(FatalIOError);
}
}
// Read slave data
for (const int proci : UPstream::subProcs(comm)) for (const int proci : UPstream::subProcs(comm))
{ {
List<char> elems(is); List<char> elems;
is.fatalCheck("read(Istream&) : reading entry"); decomposedBlockData::readBlockEntry(is, elems);
UOPstream os(proci, pBufs); UOPstream os(proci, pBufs);
os << elems; os << elems;
@ -567,24 +544,24 @@ Foam::autoPtr<Foam::ISstream> Foam::decomposedBlockData::readBlocks
// Scatter master header info // Scatter master header info
int verValue; int verValue;
int fmtValue; int fmtValue;
unsigned labelByteSize; unsigned labelWidth;
unsigned scalarByteSize; unsigned scalarWidth;
if (UPstream::master(comm)) if (UPstream::master(comm))
{ {
verValue = realIsPtr().version().canonical(); verValue = realIsPtr().version().canonical();
fmtValue = static_cast<int>(realIsPtr().format()); fmtValue = static_cast<int>(realIsPtr().format());
labelByteSize = realIsPtr().labelByteSize(); labelWidth = realIsPtr().labelByteSize();
scalarByteSize = realIsPtr().scalarByteSize(); scalarWidth = realIsPtr().scalarByteSize();
} }
Pstream::scatter(verValue); //, Pstream::msgType(), comm); Pstream::scatter(verValue); //, Pstream::msgType(), comm);
Pstream::scatter(fmtValue); //, Pstream::msgType(), comm); Pstream::scatter(fmtValue); //, Pstream::msgType(), comm);
Pstream::scatter(labelByteSize); //, Pstream::msgType(), comm); Pstream::scatter(labelWidth); //, Pstream::msgType(), comm);
Pstream::scatter(scalarByteSize); //, Pstream::msgType(), comm); Pstream::scatter(scalarWidth); //, Pstream::msgType(), comm);
realIsPtr().version(IOstreamOption::versionNumber::canonical(verValue)); realIsPtr().version(IOstreamOption::versionNumber::canonical(verValue));
realIsPtr().format(IOstreamOption::streamFormat(fmtValue)); realIsPtr().format(IOstreamOption::streamFormat(fmtValue));
realIsPtr().setLabelByteSize(labelByteSize); realIsPtr().setLabelByteSize(labelWidth);
realIsPtr().setScalarByteSize(scalarByteSize); realIsPtr().setScalarByteSize(scalarWidth);
word name(headerIO.name()); word name(headerIO.name());
Pstream::scatter(name, Pstream::msgType(), comm); Pstream::scatter(name, Pstream::msgType(), comm);
@ -656,8 +633,8 @@ void Foam::decomposedBlockData::gatherSlaveData
{ {
const label numProcs = UPstream::nProcs(comm); const label numProcs = UPstream::nProcs(comm);
sliceSizes.setSize(numProcs, 0); sliceSizes.resize(numProcs, 0);
sliceOffsets.setSize(numProcs+1, 0); sliceOffsets.resize(numProcs+1, 0);
int totalSize = 0; int totalSize = 0;
label proci = startProc; label proci = startProc;
@ -741,8 +718,8 @@ bool Foam::decomposedBlockData::writeBlocks
( (
const label comm, const label comm,
autoPtr<OSstream>& osPtr, autoPtr<OSstream>& osPtr,
List<std::streamoff>& start, List<std::streamoff>& blockOffset,
const UList<char>& data, const UList<char>& masterData,
const labelUList& recvSizes, const labelUList& recvSizes,
const PtrList<SubList<char>>& slaveData, const PtrList<SubList<char>>& slaveData,
@ -755,7 +732,7 @@ bool Foam::decomposedBlockData::writeBlocks
{ {
Pout<< "decomposedBlockData::writeBlocks:" Pout<< "decomposedBlockData::writeBlocks:"
<< " stream:" << (osPtr ? osPtr->name() : "invalid") << " stream:" << (osPtr ? osPtr->name() : "invalid")
<< " data:" << data.size() << " data:" << masterData.size()
<< " (master only) slaveData:" << slaveData.size() << " (master only) slaveData:" << slaveData.size()
<< " commsType:" << Pstream::commsTypeNames[commsType] << endl; << " commsType:" << Pstream::commsTypeNames[commsType] << endl;
} }
@ -764,35 +741,43 @@ bool Foam::decomposedBlockData::writeBlocks
bool ok = true; bool ok = true;
// Write master data
if (UPstream::master(comm))
{
blockOffset.resize(nProcs);
OSstream& os = *osPtr;
blockOffset[UPstream::masterNo()] =
decomposedBlockData::writeBlockEntry
(
os,
UPstream::masterNo(),
masterData
);
ok = os.good();
}
if (slaveData.size()) if (slaveData.size())
{ {
// Already have gathered the slave data. communicator only used to // Already have gathered the slave data.
// check who is the master
if (UPstream::master(comm)) if (UPstream::master(comm))
{ {
// Master data already written ...
OSstream& os = *osPtr; OSstream& os = *osPtr;
start.setSize(nProcs);
// Write master data
{
os << nl << "// Processor" << UPstream::masterNo() << nl;
start[UPstream::masterNo()] = os.stdStream().tellp();
os << data;
}
// Write slaves // Write slaves
label slaveOffset = 0;
for (label proci = 1; proci < nProcs; ++proci) for (label proci = 1; proci < nProcs; ++proci)
{ {
os << nl << nl << "// Processor" << proci << nl; blockOffset[proci] =
start[proci] = os.stdStream().tellp(); decomposedBlockData::writeBlockEntry
(
os << slaveData[proci]; os,
slaveOffset += recvSizes[proci]; proci,
slaveData[proci]
);
} }
ok = os.good(); ok = os.good();
@ -802,21 +787,14 @@ bool Foam::decomposedBlockData::writeBlocks
{ {
if (UPstream::master(comm)) if (UPstream::master(comm))
{ {
start.setSize(nProcs); // Master data already written ...
OSstream& os = *osPtr; OSstream& os = *osPtr;
// Write master data // Receive and write slaves
{ DynamicList<char> elems;
os << nl << "// Processor" << UPstream::masterNo() << nl;
start[UPstream::masterNo()] = os.stdStream().tellp();
os << data;
}
// Write slaves
List<char> elems;
for (label proci = 1; proci < nProcs; ++proci) for (label proci = 1; proci < nProcs; ++proci)
{ {
elems.setSize(recvSizes[proci]); elems.resize(recvSizes[proci]);
IPstream::read IPstream::read
( (
UPstream::commsTypes::scheduled, UPstream::commsTypes::scheduled,
@ -827,9 +805,13 @@ bool Foam::decomposedBlockData::writeBlocks
comm comm
); );
os << nl << nl << "// Processor" << proci << nl; blockOffset[proci] =
start[proci] = os.stdStream().tellp(); decomposedBlockData::writeBlockEntry
os << elems; (
os,
proci,
elems
);
} }
ok = os.good(); ok = os.good();
@ -840,8 +822,8 @@ bool Foam::decomposedBlockData::writeBlocks
( (
UPstream::commsTypes::scheduled, UPstream::commsTypes::scheduled,
UPstream::masterNo(), UPstream::masterNo(),
data.cdata(), masterData.cdata(),
data.size_bytes(), masterData.size_bytes(),
Pstream::msgType(), Pstream::msgType(),
comm comm
); );
@ -849,18 +831,7 @@ bool Foam::decomposedBlockData::writeBlocks
} }
else else
{ {
// Write master data // Master data already written ...
if (UPstream::master(comm))
{
start.setSize(nProcs);
OSstream& os = *osPtr;
os << nl << "// Processor" << UPstream::masterNo() << nl;
start[UPstream::masterNo()] = os.stdStream().tellp();
os << data;
}
// Find out how many processor can be received into // Find out how many processor can be received into
// maxMasterFileBufferSize // maxMasterFileBufferSize
@ -895,11 +866,11 @@ bool Foam::decomposedBlockData::writeBlocks
gatherSlaveData gatherSlaveData
( (
comm, comm,
data, masterData,
recvSizes, recvSizes,
startProc, // startProc, startProc, // startProc,
nSendProcs, // nProcs, nSendProcs, // nProcs,
sliceOffsets, sliceOffsets,
recvData recvData
@ -914,18 +885,22 @@ bool Foam::decomposedBlockData::writeBlocks
( (
label proci = startProc; label proci = startProc;
proci < startProc+nSendProcs; proci < startProc+nSendProcs;
proci++ ++proci
) )
{ {
os << nl << nl << "// Processor" << proci << nl; SubList<char> dataSlice
start[proci] = os.stdStream().tellp(); (
recvData,
sliceOffsets[proci+1]-sliceOffsets[proci],
sliceOffsets[proci]
);
os << blockOffset[proci] =
SubList<char> decomposedBlockData::writeBlockEntry
( (
recvData, os,
sliceOffsets[proci+1]-sliceOffsets[proci], proci,
sliceOffsets[proci] dataSlice
); );
} }
} }
@ -960,15 +935,12 @@ bool Foam::decomposedBlockData::read()
IOobject::readHeader(*isPtr); IOobject::readHeader(*isPtr);
} }
List<char>& data = *this; return readBlocks(comm_, isPtr, contentData_, commsType_);
return readBlocks(comm_, isPtr, data, commsType_);
} }
bool Foam::decomposedBlockData::writeData(Ostream& os) const bool Foam::decomposedBlockData::writeData(Ostream& os) const
{ {
const List<char>& data = *this;
IOobject io(*this); IOobject io(*this);
IOstreamOption streamOpt(os); IOstreamOption streamOpt(os);
@ -978,7 +950,7 @@ bool Foam::decomposedBlockData::writeData(Ostream& os) const
// Re-read my own data to find out the header information // Re-read my own data to find out the header information
if (Pstream::master(comm_)) if (Pstream::master(comm_))
{ {
UIListStream headerStream(data); UIListStream headerStream(contentData_);
io.readHeader(headerStream); io.readHeader(headerStream);
verValue = headerStream.version().canonical(); verValue = headerStream.version().canonical();
@ -1008,7 +980,7 @@ bool Foam::decomposedBlockData::writeData(Ostream& os) const
decomposedBlockData::writeHeader decomposedBlockData::writeHeader
( (
os, os,
streamOpt, streamOpt, // streamOpt for data
io.headerClassName(), io.headerClassName(),
io.note(), io.note(),
masterLocation, masterLocation,
@ -1020,12 +992,12 @@ bool Foam::decomposedBlockData::writeData(Ostream& os) const
if (isA<OFstream>(os)) if (isA<OFstream>(os))
{ {
// Serial file output - can use writeRaw() // Serial file output - can use writeRaw()
os.writeRaw(data.cdata(), data.size_bytes()); os.writeRaw(contentData_.cdata(), contentData_.size_bytes());
} }
else else
{ {
// Other cases are less fortunate, and no std::string_view // Other cases are less fortunate, and no std::string_view
std::string str(data.cdata(), data.size_bytes()); std::string str(contentData_.cdata(), contentData_.size_bytes());
os.writeQuoted(str, false); os.writeQuoted(str, false);
} }
@ -1044,29 +1016,33 @@ bool Foam::decomposedBlockData::writeObject
const bool valid const bool valid
) const ) const
{ {
// Always write BINARY
streamOpt.format(IOstream::BINARY);
autoPtr<OSstream> osPtr; autoPtr<OSstream> osPtr;
if (UPstream::master(comm_)) if (UPstream::master(comm_))
{ {
// Note: always write binary. These are strings so readable anyway. // Note: always write binary. These are strings so readable anyway.
// They have already be tokenised on the sending side. // They have already be tokenised on the sending side.
osPtr.reset(new OFstream(objectPath(), streamOpt));
IOobject::writeHeader(*osPtr); osPtr.reset(new OFstream(objectPath(), IOstreamOption::BINARY));
decomposedBlockData::writeHeader
(
*osPtr,
streamOpt, // streamOpt for data
static_cast<const IOobject&>(*this)
);
} }
labelList recvSizes; labelList recvSizes;
gather(comm_, label(this->size_bytes()), recvSizes); gather(comm_, label(contentData_.size_bytes()), recvSizes);
List<std::streamoff> start; List<std::streamoff> blockOffsets;
PtrList<SubList<char>> slaveData; // dummy slave data PtrList<SubList<char>> slaveData; // dummy slave data
return writeBlocks return writeBlocks
( (
comm_, comm_,
osPtr, osPtr,
start, blockOffsets,
*this, contentData_,
recvSizes, recvSizes,
slaveData, slaveData,
commsType_ commsType_
@ -1074,51 +1050,4 @@ bool Foam::decomposedBlockData::writeObject
} }
Foam::label Foam::decomposedBlockData::numBlocks(const fileName& fName)
{
label nBlocks = 0;
IFstream is(fName);
is.fatalCheck(FUNCTION_NAME);
if (!is.good())
{
return nBlocks;
}
// FoamFile header
token firstToken(is);
if (is.good() && firstToken.isWord("FoamFile"))
{
dictionary headerDict(is);
is.version(headerDict.get<token>("version"));
is.format(headerDict.get<word>("format"));
// Obtain number of blocks directly
if (headerDict.readIfPresent("blocks", nBlocks))
{
return nBlocks;
}
}
// Fallback to brute force read of each data block
List<char> data;
while (is.good())
{
token sizeToken(is);
if (!sizeToken.isLabel())
{
return nBlocks;
}
is.putBack(sizeToken);
is >> data;
nBlocks++;
}
return nBlocks;
}
// ************************************************************************* // // ************************************************************************* //

161
src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.H
View File

@ -28,10 +28,56 @@ Class
Foam::decomposedBlockData Foam::decomposedBlockData
Description Description
decomposedBlockData is a List<char> with IO on the master processor only. The decomposedBlockData comprise a \c List\<char\> for each output
processor, typically with IO on the master processor only.
For decomposedBlockData, we make a distinction between the container
description and the individual block contents.
The \b FoamFile header specifies the container characteristics and thus
has \c class = \c %decomposedBlockData and normally \c format = \c binary.
This description refers to the \em entire file container, not the
individual blocks.
Each processor block is simply a binary chunk of characters and the
first block also contains the header description for all of the blocks.
For example,
\verbatim
FoamFile
{
version 2.0;
format binary;
arch "LSB;label=32;scalar=64";
class decomposedBlockData;
location "constant/polyMesh";
object points;
}
// processor0
NCHARS
(FoamFile
{
version 2.0;
format ascii;
arch "LSB;label=32;scalar=64";
class vectorField;
location "constant/polyMesh";
object points;
}
...content...
)
// processor1
NCHARS
(...content...)
...
\endverbatim
SourceFiles SourceFiles
decomposedBlockData.C decomposedBlockData.C
decomposedBlockDataHeader.C
\*---------------------------------------------------------------------------*/ \*---------------------------------------------------------------------------*/
@ -53,9 +99,22 @@ namespace Foam
class decomposedBlockData class decomposedBlockData
: :
public regIOobject, public regIOobject
public List<char>
{ {
// Private Functions
//- Helper: write content for FoamFile IOobject header
static void writeHeaderContent
(
Ostream& os,
IOstreamOption streamOptContainer,
const word& objectType,
const string& note,
const fileName& location,
const word& objectName
);
protected: protected:
// Protected Data // Protected Data
@ -66,11 +125,14 @@ protected:
//- Communicator for all parallel comms //- Communicator for all parallel comms
const label comm_; const label comm_;
//- The block content
List<char> contentData_;
// Protected Member Functions // Protected Member Functions
//- Helper: determine number of processors whose recvSizes fits //- Helper: determine number of processors whose recvSizes fits
// ito maxBufferSize //- into maxBufferSize
static label calcNumProcs static label calcNumProcs
( (
const label comm, const label comm,
@ -84,7 +146,7 @@ protected:
( (
const label comm, const label comm,
autoPtr<ISstream>& isPtr, autoPtr<ISstream>& isPtr,
List<char>& data, List<char>& contentChars,
const UPstream::commsTypes commsType const UPstream::commsTypes commsType
); );
@ -105,30 +167,12 @@ public:
const UPstream::commsTypes = UPstream::commsTypes::scheduled const UPstream::commsTypes = UPstream::commsTypes::scheduled
); );
//- Construct given an IOobject and for READ_IF_MODIFIED a List<char>
decomposedBlockData
(
const label comm,
const IOobject& io,
const UList<char>& list,
const UPstream::commsTypes = UPstream::commsTypes::scheduled
);
//- Construct by transferring the List contents
decomposedBlockData
(
const label comm,
const IOobject& io,
List<char>&& list,
const UPstream::commsTypes = UPstream::commsTypes::scheduled
);
//- Destructor //- Destructor
virtual ~decomposedBlockData() = default; virtual ~decomposedBlockData() = default;
// Member functions // Member Functions
//- Read object //- Read object
virtual bool read(); virtual bool read();
@ -147,30 +191,71 @@ public:
// Helpers // Helpers
//- Read header. Call only on master. //- True if object type is a known collated type
static bool readMasterHeader(IOobject&, Istream&); static bool isCollatedType(const word& objectType);
//- True if object header class is a known collated type
static bool isCollatedType(const IOobject& io);
//- Read header as per IOobject with additional handling of
//- decomposedBlockData
static bool readHeader(IOobject& io, Istream& is);
//- Helper: write FoamFile IOobject header //- Helper: write FoamFile IOobject header
static void writeHeader static void writeHeader
( (
Ostream& os, Ostream& os,
IOstreamOption streamOpt, IOstreamOption streamOptContainer,
const word& objectType, const word& objectType,
const string& note, const string& note,
const fileName& location, const fileName& location,
const word& objectName const word& objectName
); );
//- Helper: write FoamFile IOobject header
static void writeHeader
(
Ostream& os,
IOstreamOption streamOptData,
const IOobject& io
);
//- Helper: read block of (binary) character data
static bool readBlockEntry
(
Istream& is,
List<char>& charData
);
//- Helper: write block of (binary) character data
static std::streamoff writeBlockEntry
(
OSstream& os,
const label blocki,
const UList<char>& charData
);
//- Helper: write block of (binary) character data
// \return -1 on error
static std::streamoff writeBlockEntry
(
OSstream& os,
IOstreamOption streamOptData,
const regIOobject& io,
const label blocki,
const bool withLocalHeader
);
//- Read selected block (non-seeking) + header information //- Read selected block (non-seeking) + header information
static autoPtr<ISstream> readBlock static autoPtr<ISstream> readBlock
( (
const label blocki, const label blocki,
Istream& is, ISstream& is,
IOobject& headerIO IOobject& headerIO
); );
//- Read master header information (into headerIO) and return //- Read master header information (into headerIO) and return
// data in stream. Note: isPtr is only valid on master. //- data in stream. Note: isPtr is only valid on master.
static autoPtr<ISstream> readBlocks static autoPtr<ISstream> readBlocks
( (
const label comm, const label comm,
@ -190,9 +275,11 @@ public:
labelList& datas labelList& datas
); );
//- Helper: gather data from (subset of) slaves. Returns //- Helper: gather data from (subset of) slaves.
// recvData : received data //
// recvOffsets : offset in data. recvOffsets is nProcs+1 // Returns:
// - recvData : received data
// - recvOffsets : offset in data. recvOffsets is nProcs+1
static void gatherSlaveData static void gatherSlaveData
( (
const label comm, const label comm,
@ -206,13 +293,14 @@ public:
List<char>& recvData List<char>& recvData
); );
//- Write *this. Ostream only valid on master. Returns starts of //- Write *this. Ostream only valid on master.
// processor blocks // Returns offsets of processor blocks in blockOffset
static bool writeBlocks static bool writeBlocks
( (
const label comm, const label comm,
autoPtr<OSstream>& osPtr, autoPtr<OSstream>& osPtr,
List<std::streamoff>& start, List<std::streamoff>& blockOffset,
const UList<char>& masterData, const UList<char>& masterData,
const labelUList& recvSizes, const labelUList& recvSizes,
@ -223,9 +311,6 @@ public:
const UPstream::commsTypes, const UPstream::commsTypes,
const bool syncReturnState = true const bool syncReturnState = true
); );
//- Detect number of blocks in a file
static label numBlocks(const fileName& fName);
}; };

217
src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockDataHeader.C Normal file
View File

@ -0,0 +1,217 @@
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2021 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
\*---------------------------------------------------------------------------*/
#include "decomposedBlockData.H"
#include "dictionary.H"
#include "foamVersion.H"
#include "objectRegistry.H"
#include "ListStream.H"
// * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
namespace Foam
{
inline void writeSpaces(Ostream& os, label nSpaces)
{
if (nSpaces < 1)
{
nSpaces = 1;
}
while (nSpaces--)
{
os.write(char(token::SPACE));
}
}
// Similar to writeEntry, but with fewer spaces
template<class T>
inline void writeHeaderEntry(Ostream& os, const word& key, const T& value)
{
os << indent << key;
writeSpaces(os, 12 - label(key.size()));
os << value << char(token::END_STATEMENT) << nl;
}
} // End namespace Foam
// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
void Foam::decomposedBlockData::writeHeaderContent
(
Ostream& os,
IOstreamOption streamOptContainer,
const word& objectType,
const string& note,
const fileName& location,
const word& objectName
)
{
// Standard header entries
writeHeaderEntry(os, "version", streamOptContainer.version());
writeHeaderEntry(os, "format", streamOptContainer.format());
writeHeaderEntry(os, "arch", foamVersion::buildArch);
if (!note.empty())
{
writeHeaderEntry(os, "note", note);
}
if (objectType.empty())
{
// Empty type not allowed - use 'dictionary' fallback
writeHeaderEntry(os, "class", word("dictionary"));
}
else
{
writeHeaderEntry(os, "class", objectType);
}
if (!location.empty())
{
writeHeaderEntry(os, "location", location);
}
writeHeaderEntry(os, "object", objectName);
}
// * * * * * * * * * * * * * * * Members Functions * * * * * * * * * * * * * //
bool Foam::decomposedBlockData::readHeader(IOobject& io, Istream& is)
{
dictionary headerDict;
// Read the regular "FoamFile" header
bool ok = io.readHeader(headerDict, is);
if (decomposedBlockData::isCollatedType(io))
{
// Quick information - extract from "data.class"
if (headerDict.readIfPresent("data.class", io.headerClassName()))
{
return ok;
}
{
// Master-only reading of header
List<char> charData;
decomposedBlockData::readBlockEntry(is, charData);
UIListStream headerStream(charData);
headerStream.name() = is.name();
ok = io.readHeader(headerStream);
}
}
return ok;
}
void Foam::decomposedBlockData::writeHeader
(
Ostream& os,
IOstreamOption streamOptContainer,
const word& objectType,
const string& note,
const fileName& location,
const word& objectName
)
{
if (IOobject::bannerEnabled())
{
IOobject::writeBanner(os);
}
os.beginBlock("FoamFile");
decomposedBlockData::writeHeaderContent
(
os,
streamOptContainer,
objectType,
note,
location,
objectName
);
os.endBlock();
if (IOobject::bannerEnabled())
{
IOobject::writeDivider(os) << nl;
}
}
void Foam::decomposedBlockData::writeHeader
(
Ostream& os,
IOstreamOption streamOptData,
const IOobject& io
)
{
if (IOobject::bannerEnabled())
{
IOobject::writeBanner(os);
}
os.beginBlock("FoamFile");
decomposedBlockData::writeHeaderContent
(
os,
static_cast<IOstreamOption>(os), // streamOpt container
decomposedBlockData::typeName, // class
io.note(),
(io.instance()/io.db().dbDir()/io.local()), // location
io.name()
);
{
writeHeaderEntry(os, "data.format", streamOptData.format());
writeHeaderEntry(os, "data.class", io.type());
}
// Meta-data (if any)
const dictionary* metaDataDict = io.findMetaData();
if (metaDataDict && !metaDataDict->empty())
{
metaDataDict->writeEntry("meta", os);
}
os.endBlock();
if (IOobject::bannerEnabled())
{
IOobject::writeDivider(os) << nl;
}
}
// ************************************************************************* //

26
src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.C
View File

@ -58,6 +58,7 @@ bool Foam::OFstreamCollator::writeFile
Pout<< "OFstreamCollator : Writing master " << masterData.size() Pout<< "OFstreamCollator : Writing master " << masterData.size()
<< " bytes to " << fName << " bytes to " << fName
<< " using comm " << comm << endl; << " using comm " << comm << endl;
if (slaveData.size()) if (slaveData.size())
{ {
Pout<< "OFstreamCollator : Slave data" << endl; Pout<< "OFstreamCollator : Slave data" << endl;
@ -85,10 +86,10 @@ bool Foam::OFstreamCollator::writeFile
decomposedBlockData::writeHeader decomposedBlockData::writeHeader
( (
*osPtr, *osPtr,
streamOpt, streamOpt, // streamOpt for container
objectType, objectType,
"", // note "", // note
fName, // location "", // location (leave empty instead inaccurate)
fName.name() // object name fName.name() // object name
); );
} }
@ -106,12 +107,12 @@ bool Foam::OFstreamCollator::writeFile
// the master processor in order. However can be unstable // the master processor in order. However can be unstable
// for some mpi so default is non-blocking. // for some mpi so default is non-blocking.
List<std::streamoff> start; List<std::streamoff> blockOffset;
decomposedBlockData::writeBlocks decomposedBlockData::writeBlocks
( (
comm, comm,
osPtr, osPtr,
start, blockOffset,
slice, slice,
recvSizes, recvSizes,
slaveData, slaveData,
@ -141,11 +142,8 @@ bool Foam::OFstreamCollator::writeFile
{ {
sum += recv; sum += recv;
} }
// Use ostringstream to display long int (until writing these is // Use std::to_string to display long int
// supported) Pout<< " (overall " << std::to_string(sum) << ')';
std::ostringstream os;
os << sum;
Pout<< " (overall " << os.str() << ")";
} }
Pout<< " to " << fName Pout<< " to " << fName
<< " using comm " << comm << endl; << " using comm " << comm << endl;
@ -355,10 +353,10 @@ bool Foam::OFstreamCollator::write
off_t totalSize = 0; off_t totalSize = 0;
label maxLocalSize = 0; label maxLocalSize = 0;
{ {
for (label proci = 0; proci < recvSizes.size(); proci++) for (const label recvSize : recvSizes)
{ {
totalSize += recvSizes[proci]; totalSize += recvSize;
maxLocalSize = max(maxLocalSize, recvSizes[proci]); maxLocalSize = max(maxLocalSize, recvSize);
} }
Pstream::scatter(totalSize, Pstream::msgType(), localComm_); Pstream::scatter(totalSize, Pstream::msgType(), localComm_);
Pstream::scatter(maxLocalSize, Pstream::msgType(), localComm_); Pstream::scatter(maxLocalSize, Pstream::msgType(), localComm_);
@ -443,7 +441,7 @@ bool Foam::OFstreamCollator::write
( (
UPstream::commsTypes::nonBlocking, UPstream::commsTypes::nonBlocking,
proci, proci,
reinterpret_cast<char*>(slaveData[proci].data()), slaveData[proci].data(),
slaveData[proci].size_bytes(), slaveData[proci].size_bytes(),
Pstream::msgType(), Pstream::msgType(),
localComm_ localComm_
@ -458,7 +456,7 @@ bool Foam::OFstreamCollator::write
( (
UPstream::commsTypes::nonBlocking, UPstream::commsTypes::nonBlocking,
0, 0,
reinterpret_cast<const char*>(slice.cdata()), slice.cdata(),
slice.size_bytes(), slice.size_bytes(),
Pstream::msgType(), Pstream::msgType(),
localComm_ localComm_

12
src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.H
View File

@ -71,10 +71,8 @@ class OFstreamCollator
{ {
// Private Class // Private Class
class writeData struct writeData
{ {
public:
const label comm_; const label comm_;
const word objectType_; const word objectType_;
const fileName pathName_; const fileName pathName_;
@ -105,18 +103,18 @@ class OFstreamCollator
append_(append) append_(append)
{} {}
//- (approximate) size of master + any optional slave data //- The (approximate) size of master + any optional slave data
off_t size() const off_t size() const
{ {
off_t sz = data_.size(); off_t totalSize = data_.size();
forAll(slaveData_, i) forAll(slaveData_, i)
{ {
if (slaveData_.set(i)) if (slaveData_.set(i))
{ {
sz += slaveData_[i].size(); totalSize += slaveData_[i].size();
} }
} }
return sz; return totalSize;
} }
}; };

410
src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C
View File

@ -74,13 +74,13 @@ namespace fileOperations
} }
// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * // // * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * //
Foam::labelList Foam::fileOperations::collatedFileOperation::ioRanks() Foam::labelList Foam::fileOperations::collatedFileOperation::ioRanks()
{ {
labelList ioRanks; labelList ioRanks;
string ioRanksString(getEnv("FOAM_IORANKS")); string ioRanksString(Foam::getEnv("FOAM_IORANKS"));
if (!ioRanksString.empty()) if (!ioRanksString.empty())
{ {
IStringStream is(ioRanksString); IStringStream is(ioRanksString);
@ -91,6 +91,102 @@ Foam::labelList Foam::fileOperations::collatedFileOperation::ioRanks()
} }
// * * * * * * * * * * * * Protected Member Functions * * * * * * * * * * * //
void Foam::fileOperations::collatedFileOperation::printBanner
(
const bool printRanks
) const
{
DetailInfo
<< "I/O : " << this->type();
if (maxThreadFileBufferSize == 0)
{
DetailInfo
<< " [unthreaded] (maxThreadFileBufferSize = 0)." << nl
<< " Writing may be slow for large file sizes."
<< endl;
}
else
{
DetailInfo
<< " [threaded] (maxThreadFileBufferSize = "
<< maxThreadFileBufferSize << ")." << nl
<< " Requires buffer large enough to collect all data"
" or thread support" << nl
<< " enabled in MPI. If MPI thread support cannot be"
" enabled, deactivate" << nl
<< " threading by setting maxThreadFileBufferSize"
" to 0 in" << nl
<< " OpenFOAM etc/controlDict" << endl;
}
if (printRanks)
{
// Information about the ranks
stringList ioRanks(Pstream::nProcs());
if (Pstream::master(comm_))
{
// Don't usually need the pid
// ioRanks[Pstream::myProcNo()] = hostName()+"."+name(pid());
ioRanks[Pstream::myProcNo()] = hostName();
}
Pstream::gatherList(ioRanks);
DynamicList<label> offsetMaster(Pstream::nProcs());
forAll(ioRanks, ranki)
{
if (!ioRanks[ranki].empty())
{
offsetMaster.append(ranki);
}
}
if (offsetMaster.size() > 1)
{
DetailInfo
<< "IO nodes:" << nl << '(' << nl;
offsetMaster.append(Pstream::nProcs());
for (label group = 1; group < offsetMaster.size(); ++group)
{
const label beg = offsetMaster[group-1];
const label end = offsetMaster[group];
DetailInfo
<< " (" << ioRanks[beg].c_str() << ' '
<< (end-beg) << ')' << nl;
}
DetailInfo
<< ')' << nl;
}
}
if
(
regIOobject::fileModificationChecking
== regIOobject::inotifyMaster
)
{
WarningInFunction
<< "Resetting fileModificationChecking to inotify" << endl;
}
if
(
regIOobject::fileModificationChecking
== regIOobject::timeStampMaster
)
{
WarningInFunction
<< "Resetting fileModificationChecking to timeStamp" << endl;
}
}
bool Foam::fileOperations::collatedFileOperation::isMasterRank bool Foam::fileOperations::collatedFileOperation::isMasterRank
( (
const label proci const label proci
@ -121,9 +217,9 @@ bool Foam::fileOperations::collatedFileOperation::appendObject
IOstreamOption streamOpt IOstreamOption streamOpt
) const ) const
{ {
// Append to processors/ file // Append to processorsNN/ file
label proci = detectProcessorPath(io.objectPath()); const label proci = detectProcessorPath(io.objectPath());
if (debug) if (debug)
{ {
@ -132,59 +228,15 @@ bool Foam::fileOperations::collatedFileOperation::appendObject
<< " appending processor " << proci << " appending processor " << proci
<< " data to " << pathName << endl; << " data to " << pathName << endl;
} }
if (proci == -1) if (proci == -1)
{ {
FatalErrorInFunction FatalErrorInFunction
<< "Not a valid processor path " << pathName << "Invalid processor path: " << pathName
<< exit(FatalError); << exit(FatalError);
} }
const bool isMaster = isMasterRank(proci); const bool isMaster = isMasterRank(proci);
// Determine local rank (offset) if the pathName is a per-rank one
label localProci = proci;
{
fileName path, procDir, local;
procRangeType group;
label nProcs;
splitProcessorPath(pathName, path, procDir, local, group, nProcs);
// The local rank (offset)
if (!group.empty())
{
localProci = proci - group.start();
}
}
// Create string from all data to write
string buf;
{
OStringStream os(streamOpt);
if (isMaster)
{
if (!io.writeHeader(os))
{
return false;
}
}
// Write the data to the Ostream
if (!io.writeData(os))
{
return false;
}
if (isMaster)
{
IOobject::writeEndDivider(os);
}
buf = os.str();
}
// Note: cannot do append + compression. This is a limitation // Note: cannot do append + compression. This is a limitation
// of ogzstream (or rather most compressed formats) // of ogzstream (or rather most compressed formats)
@ -204,26 +256,20 @@ bool Foam::fileOperations::collatedFileOperation::appendObject
if (isMaster) if (isMaster)
{ {
decomposedBlockData::writeHeader decomposedBlockData::writeHeader(os, streamOpt, io);
(
os,
static_cast<IOstreamOption>(os),
decomposedBlockData::typeName, // class
"", // note
pathName, // location
pathName.name() // object
);
} }
// Write data std::streamoff blockOffset = decomposedBlockData::writeBlockEntry
UList<char> slice
( (
const_cast<char*>(buf.data()), os,
label(buf.size()) streamOpt,
io,
proci,
// With FoamFile header on master?
isMaster
); );
os << nl << "// Processor" << localProci << nl << slice << nl;
return os.good(); return (blockOffset >= 0) && os.good();
} }
@ -252,81 +298,9 @@ Foam::fileOperations::collatedFileOperation::collatedFileOperation
nProcs_(Pstream::nProcs()), nProcs_(Pstream::nProcs()),
ioRanks_(ioRanks()) ioRanks_(ioRanks())
{ {
verbose = (verbose && Foam::infoDetailLevel > 0); if (verbose && Foam::infoDetailLevel > 0)
if (verbose)
{ {
DetailInfo this->printBanner(ioRanks_.size());
<< "I/O : " << typeName
<< " (maxThreadFileBufferSize " << maxThreadFileBufferSize
<< ')' << endl;
if (maxThreadFileBufferSize == 0)
{
DetailInfo
<< " Threading not activated "
"since maxThreadFileBufferSize = 0." << nl
<< " Writing may run slowly for large file sizes."
<< endl;
}
else
{
DetailInfo
<< " Threading activated "
"since maxThreadFileBufferSize > 0." << nl
<< " Requires large enough buffer to collect all data"
" or thread support " << nl
<< " enabled in MPI. If thread support cannot be "
"enabled, deactivate" << nl
<< " threading by setting maxThreadFileBufferSize "
"to 0 in" << nl
<< " OpenFOAM etc/controlDict"
<< endl;
}
if (ioRanks_.size())
{
// Print a bit of information
stringList ioRanks(Pstream::nProcs());
if (Pstream::master(comm_))
{
ioRanks[Pstream::myProcNo()] = hostName()+"."+name(pid());
}
Pstream::gatherList(ioRanks);
DetailInfo
<< " IO nodes:" << nl;
for (const string& ranks : ioRanks)
{
if (!ranks.empty())
{
DetailInfo
<< " " << ranks << nl;
}
}
}
if
(
regIOobject::fileModificationChecking
== regIOobject::inotifyMaster
)
{
WarningInFunction
<< "Resetting fileModificationChecking to inotify" << endl;
}
if
(
regIOobject::fileModificationChecking
== regIOobject::timeStampMaster
)
{
WarningInFunction
<< "Resetting fileModificationChecking to timeStamp" << endl;
}
} }
} }
@ -345,56 +319,9 @@ Foam::fileOperations::collatedFileOperation::collatedFileOperation
nProcs_(Pstream::nProcs()), nProcs_(Pstream::nProcs()),
ioRanks_(ioRanks) ioRanks_(ioRanks)
{ {
verbose = (verbose && Foam::infoDetailLevel > 0); if (verbose && Foam::infoDetailLevel > 0)
if (verbose)
{ {
DetailInfo this->printBanner(ioRanks_.size());
<< "I/O : " << typeName
<< " (maxThreadFileBufferSize " << maxThreadFileBufferSize
<< ')' << endl;
if (maxThreadFileBufferSize == 0)
{
DetailInfo
<< " Threading not activated "
"since maxThreadFileBufferSize = 0." << nl
<< " Writing may run slowly for large file sizes."
<< endl;
}
else
{
DetailInfo
<< " Threading activated "
"since maxThreadFileBufferSize > 0." << nl
<< " Requires large enough buffer to collect all data"
" or thread support " << nl
<< " enabled in MPI. If thread support cannot be "
"enabled, deactivate" << nl
<< " threading by setting maxThreadFileBufferSize "
"to 0 in the OpenFOAM etc/controlDict" << nl
<< endl;
}
if
(
regIOobject::fileModificationChecking
== regIOobject::inotifyMaster
)
{
WarningInFunction
<< "Resetting fileModificationChecking to inotify" << endl;
}
if
(
regIOobject::fileModificationChecking
== regIOobject::timeStampMaster
)
{
WarningInFunction
<< "Resetting fileModificationChecking to timeStamp" << endl;
}
} }
} }
@ -473,23 +400,22 @@ bool Foam::fileOperations::collatedFileOperation::writeObject
valid valid
); );
// If any of these fail, return (leave error handling to Ostream class) // If any of these fail, return
if (!os.good()) // (leave error handling to Ostream class)
{
return false;
}
if (!io.writeHeader(os))
{
return false;
}
// Write the data to the Ostream
if (!io.writeData(os))
{
return false;
}
IOobject::writeEndDivider(os);
return true; const bool ok =
(
os.good()
&& io.writeHeader(os)
&& io.writeData(os)
);
if (ok)
{
IOobject::writeEndDivider(os);
}
return ok;
} }
else else
{ {
@ -517,24 +443,22 @@ bool Foam::fileOperations::collatedFileOperation::writeObject
valid valid
); );
// If any of these fail, return (leave error handling to Ostream // If any of these fail, return
// class) // (leave error handling to Ostream class)
if (!os.good())
{
return false;
}
if (!io.writeHeader(os))
{
return false;
}
// Write the data to the Ostream
if (!io.writeData(os))
{
return false;
}
IOobject::writeEndDivider(os);
return true; const bool ok =
(
os.good()
&& io.writeHeader(os)
&& io.writeData(os)
);
if (ok)
{
IOobject::writeEndDivider(os);
}
return ok;
} }
else if (!Pstream::parRun()) else if (!Pstream::parRun())
{ {
@ -553,7 +477,7 @@ bool Foam::fileOperations::collatedFileOperation::writeObject
{ {
// Re-check static maxThreadFileBufferSize variable to see // Re-check static maxThreadFileBufferSize variable to see
// if needs to use threading // if needs to use threading
bool useThread = (maxThreadFileBufferSize > 0); const bool useThread = (maxThreadFileBufferSize > 0);
if (debug) if (debug)
{ {
@ -576,27 +500,25 @@ bool Foam::fileOperations::collatedFileOperation::writeObject
useThread useThread
); );
// If any of these fail, return (leave error handling to Ostream // If any of these fail, return
// class) // (leave error handling to Ostream class)
if (!os.good())
{ bool ok = os.good();
return false;
}
if (Pstream::master(comm_) && !io.writeHeader(os))
{
return false;
}
// Write the data to the Ostream
if (!io.writeData(os))
{
return false;
}
if (Pstream::master(comm_)) if (Pstream::master(comm_))
{ {
IOobject::writeEndDivider(os); // Suppress comment banner
const bool old = IOobject::bannerEnabled(false);
ok = ok && io.writeHeader(os);
IOobject::bannerEnabled(old);
} }
return true; ok = ok && io.writeData(os);
// No end divider for collated output
return ok;
} }
} }
} }
@ -629,7 +551,7 @@ Foam::word Foam::fileOperations::collatedFileOperation::processorsDir
{ {
procDir += procDir +=
+ "_" + "_"
+ Foam::name(procs[0]) + Foam::name(procs.first())
+ "-" + "-"
+ Foam::name(procs.last()); + Foam::name(procs.last());
} }
@ -649,19 +571,19 @@ Foam::word Foam::fileOperations::collatedFileOperation::processorsDir
// Find lowest io rank // Find lowest io rank
label minProc = 0; label minProc = 0;
label maxProc = nProcs_-1; label maxProc = nProcs_-1;
forAll(ioRanks_, i) for (const label ranki : ioRanks_)
{ {
if (ioRanks_[i] >= nProcs_) if (ranki >= nProcs_)
{ {
break; break;
} }
else if (ioRanks_[i] <= proci) else if (ranki <= proci)
{ {
minProc = ioRanks_[i]; minProc = ranki;
} }
else else
{ {
maxProc = ioRanks_[i]-1; maxProc = ranki-1;
break; break;
} }
} }

12
src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.H
View File

@ -65,7 +65,7 @@ class collatedFileOperation
{ {
protected: protected:
// Protected data // Protected Data
//- Any communicator allocated by me //- Any communicator allocated by me
const label myComm_; const label myComm_;
@ -82,15 +82,19 @@ protected:
const labelList ioRanks_; const labelList ioRanks_;
// Private Member Functions // Protected Member Functions
//- Retrieve list of IO ranks from FOAM_IORANKS env variable
static labelList ioRanks(); static labelList ioRanks();
//- Print banner information, optionally with io ranks
void printBanner(const bool printRanks = false) const;
//- Is proci master of communicator (in parallel) or master of //- Is proci master of communicator (in parallel) or master of
// the io ranks (non-parallel) //- the io ranks (non-parallel)
bool isMasterRank(const label proci) const; bool isMasterRank(const label proci) const;
//- Append to processors/ file //- Append to processorsNN/ file
bool appendObject bool appendObject
( (
const regIOobject& io, const regIOobject& io,

26
src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.C
View File

@ -6,6 +6,7 @@
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2017-2018 OpenFOAM Foundation Copyright (C) 2017-2018 OpenFOAM Foundation
Copyright (C) 2021 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
@ -137,31 +138,14 @@ Foam::fileOperations::hostCollatedFileOperation::hostCollatedFileOperation
UPstream::worldComm, UPstream::worldComm,
subRanks(Pstream::nProcs()) subRanks(Pstream::nProcs())
), ),
(Pstream::parRun() ? labelList(0) : ioRanks()), // processor dirs (Pstream::parRun() ? labelList() : ioRanks()), // processor dirs
typeName, typeName,
verbose false // verbose
) )
{ {
verbose = (verbose && Foam::infoDetailLevel > 0); if (verbose && Foam::infoDetailLevel > 0)
if (verbose)
{ {
// Print a bit of information this->printBanner(ioRanks_.size());
stringList ioRanks(Pstream::nProcs());
if (Pstream::master(comm_))
{
ioRanks[Pstream::myProcNo()] = hostName()+"."+name(pid());
}
Pstream::gatherList(ioRanks);
Info<< " IO nodes:" << nl;
for (const string& ranks : ioRanks)
{
if (!ranks.empty())
{
Info<< " " << ranks << nl;
}
}
} }
} }

53
src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2017-2018 OpenFOAM Foundation Copyright (C) 2017-2018 OpenFOAM Foundation
Copyright (C) 2019-2020 OpenCFD Ltd. Copyright (C) 2019-2021 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
@ -636,23 +636,20 @@ bool Foam::fileOperation::writeObject
OSstream& os = *osPtr; OSstream& os = *osPtr;
// If any of these fail, return (leave error handling to Ostream class) // If any of these fail, return (leave error handling to Ostream class)
if (!os.good())
const bool ok =
(
os.good()
&& io.writeHeader(os)
&& io.writeData(os)
);
if (ok)
{ {
return false; IOobject::writeEndDivider(os);
} }
if (!io.writeHeader(os)) return ok;
{
return false;
}
// Write the data to the Ostream
if (!io.writeData(os))
{
return false;
}
IOobject::writeEndDivider(os);
} }
return true; return true;
} }
@ -1087,29 +1084,15 @@ Foam::label Foam::fileOperation::nProcs
} }
nProcs = maxProc+1; nProcs = maxProc+1;
if (nProcs == 0 && Foam::isDir(dir/processorsBaseDir)) if (nProcs == 0 && Foam::isDir(dir/processorsBaseDir))
{ {
fileName pointsFile WarningInFunction
( << "Defunct collated naming: " << processorsBaseDir << nl
dir << "Manually rename with the decomposition number. Eg," << nl << nl
/processorsBaseDir << " mv processors processors16" << nl << nl
/"constant" << "...returning 1" << endl;
/local
/polyMesh::meshSubDir
/"points"
);
if (Foam::isFile(pointsFile)) nProcs = 1;
{
nProcs = decomposedBlockData::numBlocks(pointsFile);
}
else
{
WarningInFunction << "Cannot read file " << pointsFile
<< " to determine the number of decompositions."
<< " Returning 1" << endl;
}
} }
} }
Pstream::scatter(nProcs, Pstream::msgType(), comm_); Pstream::scatter(nProcs, Pstream::msgType(), comm_);

65
src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C
View File

@ -832,7 +832,7 @@ masterUncollatedFileOperationInitialise(int& argc, char**& argv)
if (argv[i] == s) if (argv[i] == s)
{ {
index = i; index = i;
setEnv("FOAM_IORANKS", argv[i+1], true); Foam::setEnv("FOAM_IORANKS", argv[i+1], true);
break; break;
} }
} }
@ -1782,12 +1782,8 @@ bool Foam::fileOperations::masterUncollatedFileOperation::readHeader
if (is.good()) if (is.good())
{ {
ok = io.readHeader(is); // Regular header or from decomposed data
if (io.headerClassName() == decomposedBlockData::typeName) ok = decomposedBlockData::readHeader(io, is);
{
// Read the header inside the container (master data)
ok = decomposedBlockData::readMasterHeader(io, is);
}
} }
} }
} }
@ -1831,22 +1827,8 @@ bool Foam::fileOperations::masterUncollatedFileOperation::readHeader
if (is.good()) if (is.good())
{ {
result[proci] = io.readHeader(is); result[proci] =
if decomposedBlockData::readHeader(io, is);
(
io.headerClassName()
== decomposedBlockData::typeName
)
{
// Read the header inside the container
// (master data)
result[proci] = decomposedBlockData::
readMasterHeader
(
io,
is
);
}
headerClassName[proci] = io.headerClassName(); headerClassName[proci] = io.headerClassName();
note[proci] = io.note(); note[proci] = io.note();
} }
@ -1905,7 +1887,7 @@ Foam::fileOperations::masterUncollatedFileOperation::readStream
// have no file to read from. This will only happen when using // have no file to read from. This will only happen when using
// normal writing since then the fName for the valid processors is // normal writing since then the fName for the valid processors is
// processorDDD/<instance>/.. . In case of collocated writing // processorDDD/<instance>/.. . In case of collocated writing
// the fName is already rewritten to processors/. // the fName is already rewritten to processorsNN/.
isPtr.reset(new IFstream(fName)); isPtr.reset(new IFstream(fName));
@ -1914,11 +1896,9 @@ Foam::fileOperations::masterUncollatedFileOperation::readStream
// Read header data (on copy) // Read header data (on copy)
headerIO.readHeader(*isPtr); headerIO.readHeader(*isPtr);
if (headerIO.headerClassName() == decomposedBlockData::typeName) isCollated = decomposedBlockData::isCollatedType(headerIO);
{
isCollated = true; if (!isCollated && !Pstream::parRun())
}
else if (!Pstream::parRun())
{ {
// Short circuit: non-collated format. No parallel bits. // Short circuit: non-collated format. No parallel bits.
// Copy header and return. // Copy header and return.
@ -2206,25 +2186,20 @@ bool Foam::fileOperations::masterUncollatedFileOperation::writeObject
OSstream& os = *osPtr; OSstream& os = *osPtr;
// If any of these fail, return (leave error handling to Ostream class) // If any of these fail, return (leave error handling to Ostream class)
if (!os.good())
const bool ok =
(
os.good()
&& io.writeHeader(os)
&& io.writeData(os)
);
if (ok)
{ {
return false; IOobject::writeEndDivider(os);
} }
if (!io.writeHeader(os)) return ok;
{
return false;
}
// Write the data to the Ostream
if (!io.writeData(os))
{
return false;
}
IOobject::writeEndDivider(os);
return true;
} }

12
src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C
View File

@ -508,13 +508,8 @@ bool Foam::fileOperations::uncollatedFileOperation::readHeader
return false; return false;
} }
bool ok = io.readHeader(*isPtr); // Regular header or from decomposed data
bool ok = decomposedBlockData::readHeader(io, *isPtr);
if (io.headerClassName() == decomposedBlockData::typeName)
{
// Read the header inside the container (master data)
ok = decomposedBlockData::readMasterHeader(io, *isPtr);
}
if (debug) if (debug)
{ {
@ -570,8 +565,9 @@ Foam::fileOperations::uncollatedFileOperation::readStream
<< exit(FatalIOError); << exit(FatalIOError);
} }
if (io.headerClassName() != decomposedBlockData::typeName) if (!decomposedBlockData::isCollatedType(io))
{ {
// Short circuit: non-collated format.
return isPtr; return isPtr;
} }
else else