zhyang-dev/openfoam
1
0
Fork 0
mirror of https://develop.openfoam.com/Development/openfoam.git synced 2025-11-28 03:28:01 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

ENH: additional routines for reading/writing/parsing IOObject headers

Browse Source 
- support selective enable/disable of the file banner.

ENH: improve code isolation for decomposedBlockData

- use readBlockEntry/writeBlockEntry to encapsulate the IO handling,
  which ensures more consistency

- new decomposedBlockData::readHeader for chaining into the
  block header information.

- remove unused constructors for decomposedBlockData

ENH: minor cleanup of collated fileOperations
This commit is contained in:
Mark Olesen
2021-03-05 14:14:27 +01:00
committed by Andrew Heather
parent e8cf2a2c62
commit 0c985edfc8
19 changed files with 1282 additions and 916 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
applications/test/IOField/Test-IOField.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2017 OpenFOAM Foundation
Copyright (C) 2019 OpenCFD Ltd.
Copyright (C) 2019-2021 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -96,8 +96,8 @@ void writeAndRead
Pout<< "** Writing:" << writeType
<< " Reading:" << readType << endl;
autoPtr<fileOperation> writeHandler(fileOperation::New(writeType, true));
fileHandler(writeHandler);
// The write handler
fileHandler(fileOperation::New(writeType, true));
// Delete
Pout<< "Deleting:" << fileHandler().filePath(io.objectPath()) << endl;
@ -107,8 +107,8 @@ void writeAndRead
Pout<< "Writing:" << fileHandler().objectPath(io, io.name()) << endl;
doWrite<Type>(io, sz);
autoPtr<fileOperation> readHandler(fileOperation::New(readType, true));
fileHandler(readHandler);
// The read handler
fileHandler(fileOperation::New(readType, true));
// Read
IOobject readIO(io);
@ -130,8 +130,7 @@ void readIfPresent
const word& readType
)
{
autoPtr<fileOperation> readHandler(fileOperation::New(readType, true));
fileHandler(readHandler);
fileHandler(fileOperation::New(readType, true));
// Read
Pout<< "Reading:" << fileHandler().filePath(io.objectPath()) << endl;
@ -178,6 +177,7 @@ void doTests(IOobject& io, const label sz)
int main(int argc, char *argv[])
{
argList::noBanner();
argList::addBoolOption("bool", "Use bool for tests");
argList::addBoolOption("scalar", "Use scalar for tests");
argList::addBoolOption("label", "Use label for tests (default)");
@ -209,6 +209,11 @@ int main(int argc, char *argv[])
IOobject::NO_WRITE
);
{
dictionary headerDict;
io.writeHeader(headerDict, "anything", IOstreamOption());
Info<< "IOobjectHeader" << headerDict << nl;
}
label tested = 0;

2
src/OpenFOAM/Make/files
View File

@ -311,6 +311,7 @@ $(IOdictionary)/unwatchedIOdictionary.C
db/IOobjects/IOMap/IOMapName.C
db/IOobjects/decomposedBlockData/decomposedBlockData.C
db/IOobjects/decomposedBlockData/decomposedBlockDataHeader.C
db/IOobjects/GlobalIOField/GlobalIOFields.C
@ -320,6 +321,7 @@ db/IOobjects/GlobalIOList/globalIOLists.C
IOobject = db/IOobject
$(IOobject)/IOobject.C
$(IOobject)/IOobjectIO.C
$(IOobject)/IOobjectMetaData.C
$(IOobject)/IOobjectReadHeader.C
$(IOobject)/IOobjectWriteHeader.C

10
src/OpenFOAM/db/IOobject/IOobject.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2016-2020 OpenCFD Ltd.
Copyright (C) 2016-2021 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -28,7 +28,7 @@ License
#include "IOobject.H"
#include "Time.H"
#include "IFstream.H"
#include "Istream.H"
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
@ -37,6 +37,8 @@ namespace Foam
defineTypeNameAndDebug(IOobject, 0);
}
bool Foam::IOobject::bannerEnabled_(true);
char Foam::IOobject::scopeSeparator
(
#ifdef _WIN32
@ -47,7 +49,6 @@ char Foam::IOobject::scopeSeparator
#endif
);
const Foam::Enum
<
Foam::IOobject::fileCheckTypes
@ -60,7 +61,6 @@ Foam::IOobject::fileCheckTypesNames
{ fileCheckTypes::inotifyMaster, "inotifyMaster" },
});
// Default fileCheck type
Foam::IOobject::fileCheckTypes Foam::IOobject::fileModificationChecking
(
@ -460,7 +460,7 @@ Foam::IOobject::IOobject
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
const Foam::objectRegistry& Foam::IOobject::db() const
const Foam::objectRegistry& Foam::IOobject::db() const noexcept
{
return db_;
}

136
src/OpenFOAM/db/IOobject/IOobject.H
View File

@ -64,6 +64,21 @@ Description
- \par NO_WRITE
No automatic write on destruction but can be written explicitly
When serializing, the IOobject characteristics are typically written
as a \c FoamFile header, which is a sub-dictionary with the following
type of content:
\table
Property | Description | Type | Reqd | Deflt
version | The base format version | float | no | 2.0
format | The stream format (ascii/binary) | word | yes |
arch | The architecture string | string | no |
note | Descriptive note about the object | string | no |
location | The relative location of the object | string | no |
class | The type of the object | word | yes |
object | The name of the object | word | yes |
\endtable
Note
Specifying registered does not result in the IOobject itself being
registered. It is only serves as guidance for a regIOobject using it.
@ -86,6 +101,7 @@ SourceFiles
#include "fileName.H"
#include "typeInfo.H"
#include "autoPtr.H"
#include "IOstreamOption.H"
#include "InfoProxy.H"
#include "Enum.H"
@ -96,6 +112,7 @@ namespace Foam
// Forward Declarations
class Time;
class dictionary;
class objectRegistry;
/*---------------------------------------------------------------------------*\
@ -146,6 +163,12 @@ public:
private:
// Static Data Members
//- Use an output file banner, enabled by default
static bool bannerEnabled_;
// Private Data
//- Name
@ -192,6 +215,27 @@ protected:
// Protected Member Functions
//- Helper: write content for FoamFile IOobject header
//- with optional meta information.
static void writeHeaderContent
(
Ostream& os,
const IOobject& io,
const word& objectType,
const dictionary* metaDataDict = nullptr
);
//- Helper: write dictionary content for FoamFile header
//- with optional meta information.
static void writeHeaderContent
(
dictionary& os,
const IOobject& io,
const word& objectType,
IOstreamOption streamOpt,
const dictionary* metaDataDict = nullptr
);
//- Set the object state to bad
void setBad(const string& s);
@ -214,6 +258,21 @@ public:
// Static Functions
//- Status of output file banner
static bool bannerEnabled() noexcept
{
return bannerEnabled_;
}
//- Enable/disable an output file banner
// \return the previous value
static bool bannerEnabled(bool on) noexcept
{
bool old(bannerEnabled_);
bannerEnabled_ = on;
return old;
}
//- Split path into instance, local, name components
//
// The splitting behaviour is as follows:
@ -364,28 +423,28 @@ public:
// Member Functions
// General access
// General Access
//- Return the local objectRegistry
const objectRegistry& db() const;
const objectRegistry& db() const noexcept;
//- Return Time associated with the objectRegistry
const Time& time() const;
//- Return name
inline const word& name() const;
inline const word& name() const noexcept;
//- Return name of the class name read from header
inline const word& headerClassName() const;
inline const word& headerClassName() const noexcept;
//- Return non-constant access to the class name read from header
inline word& headerClassName();
inline word& headerClassName() noexcept;
//- Return the optional note
inline const string& note() const;
inline const string& note() const noexcept;
//- Return non-constant access to the optional note
inline string& note();
inline string& note() noexcept;
//- Rename
virtual void rename(const word& newName)
@ -394,16 +453,16 @@ public:
}
//- Should object created with this IOobject be registered?
inline bool registerObject() const;
inline bool registerObject() const noexcept;
//- Should object created with this IOobject be registered?
inline bool& registerObject();
inline bool& registerObject() noexcept;
//- Is object same for all processors?
inline bool globalObject() const;
inline bool globalObject() const noexcept;
//- Is object same for all processors?
inline bool& globalObject();
inline bool& globalObject() noexcept;
//- The sizeof (label) in bytes, possibly read from the header
inline unsigned labelByteSize() const noexcept;
@ -422,19 +481,25 @@ public:
inline bool isHeaderClassName() const;
// Meta-data
//- Return pointer to meta-data (if any) or nullptr
virtual const dictionary* findMetaData() const noexcept;
// Read/write options
//- The read option
inline readOption readOpt() const;
inline readOption readOpt() const noexcept;
//- Non-constant access to the read option
inline readOption& readOpt();
inline readOption& readOpt() noexcept;
//- The write option
inline writeOption writeOpt() const;
inline writeOption writeOpt() const noexcept;
//- Non-constant access to the write option
inline writeOption& writeOpt();
inline writeOption& writeOpt() noexcept;
// Path components
@ -449,11 +514,11 @@ public:
const fileName& caseName() const;
inline const fileName& instance() const;
inline const fileName& instance() const noexcept;
inline fileName& instance();
inline fileName& instance() noexcept;
inline const fileName& local() const;
inline const fileName& local() const noexcept;
//- The complete path
fileName path() const;
@ -489,9 +554,19 @@ public:
// Reading
//- Read header
//- Parse 'FoamFile' header contents and set the IOobject
//- characteristics and return the stream characteristics.
IOstreamOption parseHeader(const dictionary& headerDict);
//- Read header ('FoamFile' dictionary) and set the
//- IOobject and stream characteristics.
bool readHeader(Istream& is);
//- Read header (the 'FoamFile' dictionary) and set the
//- IOobject and stream characteristics.
// Saves the header content in the given dictionary.
bool readHeader(dictionary& headerDict, Istream& is);
//- Read header (uses typeFilePath to find file) and check its info.
// Optionally checks headerClassName against the type-name.
// When search is false, simply use the current instance,
@ -521,24 +596,31 @@ public:
//- Write the standard end file divider
static Ostream& writeEndDivider(Ostream& os);
//- Write header with current type() and arch information
//- Write header with current type()
bool writeHeader(Ostream& os) const;
//- Write header with override of type and optionally without
//- arch information in ASCII
bool writeHeader
//- Write header with override of type
bool writeHeader(Ostream& os, const word& objectType) const;
//- Write header into a dictionary with current type()
//- and given output format
void writeHeader(dictionary& dict, IOstreamOption streamOpt) const;
//- Write header into a dictionary with override of type
//- and given output format
void writeHeader
(
Ostream& os,
dictionary& dict,
const word& objectType,
const bool noArchAscii = false
IOstreamOption streamOpt
) const;
// Error Handling
inline bool good() const;
inline bool good() const noexcept;
inline bool bad() const;
inline bool bad() const noexcept;
// Info

36
src/OpenFOAM/db/IOobject/IOobjectI.H
View File

@ -67,7 +67,7 @@ inline Foam::word Foam::IOobject::scopedName
// General access
inline const Foam::word& Foam::IOobject::name() const
inline const Foam::word& Foam::IOobject::name() const noexcept
{
return name_;
}
@ -85,49 +85,49 @@ inline Foam::word Foam::IOobject::member() const
}
inline const Foam::word& Foam::IOobject::headerClassName() const
inline const Foam::word& Foam::IOobject::headerClassName() const noexcept
{
return headerClassName_;
}
inline Foam::word& Foam::IOobject::headerClassName()
inline Foam::word& Foam::IOobject::headerClassName() noexcept
{
return headerClassName_;
}
inline const Foam::string& Foam::IOobject::note() const
inline const Foam::string& Foam::IOobject::note() const noexcept
{
return note_;
}
inline Foam::string& Foam::IOobject::note()
inline Foam::string& Foam::IOobject::note() noexcept
{
return note_;
}
inline bool Foam::IOobject::registerObject() const
inline bool Foam::IOobject::registerObject() const noexcept
{
return registerObject_;
}
inline bool& Foam::IOobject::registerObject()
inline bool& Foam::IOobject::registerObject() noexcept
{
return registerObject_;
}
inline bool Foam::IOobject::globalObject() const
inline bool Foam::IOobject::globalObject() const noexcept
{
return globalObject_;
}
inline bool& Foam::IOobject::globalObject()
inline bool& Foam::IOobject::globalObject() noexcept
{
return globalObject_;
}
@ -162,25 +162,25 @@ inline bool Foam::IOobject::isHeaderClassName() const
// Read/write options
inline Foam::IOobject::readOption Foam::IOobject::readOpt() const
inline Foam::IOobject::readOption Foam::IOobject::readOpt() const noexcept
{
return rOpt_;
}
inline Foam::IOobject::readOption& Foam::IOobject::readOpt()
inline Foam::IOobject::readOption& Foam::IOobject::readOpt() noexcept
{
return rOpt_;
}
inline Foam::IOobject::writeOption Foam::IOobject::writeOpt() const
inline Foam::IOobject::writeOption Foam::IOobject::writeOpt() const noexcept
{
return wOpt_;
}
inline Foam::IOobject::writeOption& Foam::IOobject::writeOpt()
inline Foam::IOobject::writeOption& Foam::IOobject::writeOpt() noexcept
{
return wOpt_;
}
@ -188,19 +188,19 @@ inline Foam::IOobject::writeOption& Foam::IOobject::writeOpt()
// Path components
inline const Foam::fileName& Foam::IOobject::instance() const
inline const Foam::fileName& Foam::IOobject::instance() const noexcept
{
return instance_;
}
inline Foam::fileName& Foam::IOobject::instance()
inline Foam::fileName& Foam::IOobject::instance() noexcept
{
return instance_;
}
inline const Foam::fileName& Foam::IOobject::local() const
inline const Foam::fileName& Foam::IOobject::local() const noexcept
{
return local_;
}
@ -214,13 +214,13 @@ inline Foam::fileName Foam::IOobject::objectPath() const
// Error Handling
inline bool Foam::IOobject::good() const
inline bool Foam::IOobject::good() const noexcept
{
return objState_ == GOOD;
}
inline bool Foam::IOobject::bad() const
inline bool Foam::IOobject::bad() const noexcept
{
return objState_ == BAD;
}

39
src/OpenFOAM/db/IOobject/IOobjectMetaData.C Normal file
View File

@ -0,0 +1,39 @@
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2021 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
\*---------------------------------------------------------------------------*/
#include "IOobject.H"
#include "dictionary.H"
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
const Foam::dictionary* Foam::IOobject::findMetaData() const noexcept
{
return nullptr;
}
// ************************************************************************* //

88
src/OpenFOAM/db/IOobject/IOobjectReadHeader.C
View File

@ -32,7 +32,45 @@ License
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
bool Foam::IOobject::readHeader(Istream& is)
Foam::IOstreamOption Foam::IOobject::parseHeader(const dictionary& headerDict)
{
IOstreamOption streamOpt; // == (ASCII, currentVersion)
// Treat "version" as optional
{
token tok;
if (headerDict.readIfPresent("version", tok))
{
streamOpt.version(tok);
}
}
// Treat "format" as mandatory, could also as optional
streamOpt.format(headerDict.get<word>("format"));
headerClassName_ = headerDict.get<word>("class");
const word headerObject(headerDict.get<word>("object"));
// The "note" entry is optional
headerDict.readIfPresent("note", note_);
// The "arch" information may be missing
string arch;
if (headerDict.readIfPresent("arch", arch))
{
unsigned val = foamVersion::labelByteSize(arch);
if (val) sizeofLabel_ = static_cast<unsigned char>(val);
val = foamVersion::scalarByteSize(arch);
if (val) sizeofScalar_ = static_cast<unsigned char>(val);
}
return streamOpt;
}
bool Foam::IOobject::readHeader(dictionary& headerDict, Istream& is)
{
if (IOobject::debug)
{
@ -64,48 +102,21 @@ bool Foam::IOobject::readHeader(Istream& is)
if (is.good() && firstToken.isWord("FoamFile"))
{
const dictionary headerDict(is);
headerDict.read(is, false); // Read sub-dictionary content
is.version(headerDict.get<token>("version"));
is.format(headerDict.get<word>("format"));
headerClassName_ = headerDict.get<word>("class");
const word headerObject(headerDict.get<word>("object"));
if (IOobject::debug && headerObject != name())
{
IOWarningInFunction(is)
<< " object renamed from "
<< name() << " to " << headerObject
<< " for file " << is.name() << endl;
}
// The note entry is optional
headerDict.readIfPresent("note", note_);
sizeofLabel_ = sizeof(label);
sizeofScalar_ = sizeof(scalar);
// The arch information is optional
string arch;
if (headerDict.readIfPresent("arch", arch))
{
unsigned val = foamVersion::labelByteSize(arch);
if (val) sizeofLabel_ = static_cast<unsigned char>(val);
val = foamVersion::scalarByteSize(arch);
if (val) sizeofScalar_ = static_cast<unsigned char>(val);
}
IOstreamOption streamOpt = parseHeader(headerDict);
is.format(streamOpt.format());
is.version(streamOpt.version());
is.setLabelByteSize(sizeofLabel_);
is.setScalarByteSize(sizeofScalar_);
}
else
{
IOWarningInFunction(is)
<< "First token could not be read or is not the keyword 'FoamFile'"
<< nl << nl << "Check header is of the form:" << nl << endl;
<< "First token could not be read or is not 'FoamFile'"
<< nl << nl
<< "Check header is of the form:" << nl << endl;
writeHeader(Info);
@ -151,4 +162,11 @@ bool Foam::IOobject::readHeader(Istream& is)
}
bool Foam::IOobject::readHeader(Istream& is)
{
dictionary headerDict;
return IOobject::readHeader(headerDict, is);
}
// ************************************************************************* //

193
src/OpenFOAM/db/IOobject/IOobjectWriteHeader.C
View File

@ -27,10 +27,40 @@ License
\*---------------------------------------------------------------------------*/
#include "IOobject.H"
#include "dictionary.H"
#include "objectRegistry.H"
#include "foamVersion.H"
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
// * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
namespace Foam
{
inline void writeSpaces(Ostream& os, label nSpaces)
{
if (nSpaces < 1)
{
nSpaces = 1;
}
while (nSpaces--)
{
os.write(char(token::SPACE));
}
}
// Similar to writeEntry, but with fewer spaces
template<class T>
inline void writeHeaderEntry(Ostream& os, const word& key, const T& value)
{
os << indent << key;
writeSpaces(os, 12 - label(key.size()));
os << value << char(token::END_STATEMENT) << nl;
}
} // End namespace Foam
// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * //
// A banner corresponding to this:
//
@ -117,11 +147,93 @@ Foam::Ostream& Foam::IOobject::writeEndDivider(Ostream& os)
}
// * * * * * * * * * * * * Protected Member Functions * * * * * * * * * * * //
void Foam::IOobject::writeHeaderContent
(
Ostream& os,
const IOobject& io,
const word& objectType,
const dictionary* metaDataDict
)
{
// Standard header entries
writeHeaderEntry(os, "version", os.version());
writeHeaderEntry(os, "format", os.format());
writeHeaderEntry(os, "arch", foamVersion::buildArch);
if (!io.note().empty())
{
writeHeaderEntry(os, "note", io.note());
}
if (objectType.empty())
{
// Empty type not allowed - use 'dictionary' fallback
writeHeaderEntry(os, "class", word("dictionary"));
}
else
{
writeHeaderEntry(os, "class", objectType);
}
writeHeaderEntry(os, "location", io.instance()/io.db().dbDir()/io.local());
writeHeaderEntry(os, "object", io.name());
// Meta-data (if any)
if (metaDataDict && !metaDataDict->empty())
{
metaDataDict->writeEntry("meta", os);
}
}
void Foam::IOobject::writeHeaderContent
(
dictionary& dict,
const IOobject& io,
const word& objectType,
IOstreamOption streamOpt,
const dictionary* metaDataDict
)
{
// Standard header entries
dict.set("version", streamOpt.version());
dict.set("format", streamOpt.format());
dict.set("arch", foamVersion::buildArch);
if (!io.note().empty())
{
dict.set("note", io.note());
}
if (objectType.empty())
{
// Empty type not allowed - use 'dictionary' fallback
dict.set("class", word("dictionary"));
}
else
{
dict.set("class", objectType);
}
dict.set("location", io.instance()/io.db().dbDir()/io.local());
dict.set("object", io.name());
// Deep-copy of meta-data (if any)
if (metaDataDict && !metaDataDict->empty())
{
dict.add("meta", *metaDataDict);
}
}
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
bool Foam::IOobject::writeHeader
(
Ostream& os,
const word& objectType,
const bool noArchAscii
const word& objectType
) const
{
if (!os.good())
@ -133,39 +245,28 @@ bool Foam::IOobject::writeHeader
return false;
}
IOobject::writeBanner(os)
<< "FoamFile" << nl
<< '{' << nl
<< " version " << os.version() << ';' << nl
<< " format " << os.format() << ';' << nl;
if (os.format() == IOstream::BINARY || !noArchAscii)
if (IOobject::bannerEnabled())
{
// Arch information (BINARY: always, ASCII: can disable)
os << " arch " << foamVersion::buildArch << ';' << nl;
}
if (!note().empty())
{
os << " note " << note() << ';' << nl;
IOobject::writeBanner(os);
}
os << " class ";
if (objectType.empty())
{
// Empty type not allowed - use 'dictionary' fallback
os << "dictionary";
}
else
{
os << objectType;
}
os << ';' << nl;
os.beginBlock("FoamFile");
os << " location " << instance()/db().dbDir()/local() << ';' << nl
<< " object " << name() << ';' << nl
<< '}' << nl;
// Standard header entries
IOobject::writeHeaderContent
(
os,
*this,
objectType,
this->findMetaData()
);
writeDivider(os) << nl;
os.endBlock();
if (IOobject::bannerEnabled())
{
IOobject::writeDivider(os) << nl;
}
return true;
}
@ -173,7 +274,35 @@ bool Foam::IOobject::writeHeader
bool Foam::IOobject::writeHeader(Ostream& os) const
{
return writeHeader(os, type());
return IOobject::writeHeader(os, this->type());
}
void Foam::IOobject::writeHeader
(
dictionary& dict,
const word& objectType,
IOstreamOption streamOpt
) const
{
IOobject::writeHeaderContent
(
dict,
*this,
objectType,
streamOpt,
this->findMetaData()
);
}
void Foam::IOobject::writeHeader
(
dictionary& dict,
IOstreamOption streamOpt
) const
{
IOobject::writeHeader(dict, this->type(), streamOpt);
}

681
src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.C
View File

@ -34,10 +34,11 @@ License
#include "dictionary.H"
#include "objectRegistry.H"
#include "SubList.H"
#include "charList.H"
#include "labelPair.H"
#include "masterUncollatedFileOperation.H"
#include "IListStream.H"
#include "foamVersion.H"
#include "ListStream.H"
#include "StringStream.H"
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
@ -46,6 +47,30 @@ namespace Foam
defineTypeNameAndDebug(decomposedBlockData, 0);
}
// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * //
bool Foam::decomposedBlockData::isCollatedType
(
const word& objectType
)
{
return
(
objectType == decomposedBlockData::typeName
);
}
bool Foam::decomposedBlockData::isCollatedType
(
const IOobject& io
)
{
return decomposedBlockData::isCollatedType(io.headerClassName());
}
// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
Foam::decomposedBlockData::decomposedBlockData
@ -57,7 +82,8 @@ Foam::decomposedBlockData::decomposedBlockData
:
regIOobject(io),
commsType_(commsType),
comm_(comm)
comm_(comm),
contentData_()
{
// Temporary warning
if (io.readOpt() == IOobject::MUST_READ_IF_MODIFIED)
@ -82,161 +108,125 @@ Foam::decomposedBlockData::decomposedBlockData
}
Foam::decomposedBlockData::decomposedBlockData
(
const label comm,
const IOobject& io,
const UList<char>& list,
const UPstream::commsTypes commsType
)
:
regIOobject(io),
commsType_(commsType),
comm_(comm)
{
// Temporary warning
if (io.readOpt() == IOobject::MUST_READ_IF_MODIFIED)
{
WarningInFunction
<< "decomposedBlockData " << name()
<< " constructed with IOobject::MUST_READ_IF_MODIFIED"
" but decomposedBlockData does not support automatic rereading."
<< endl;
}
if
(
(
io.readOpt() == IOobject::MUST_READ
|| io.readOpt() == IOobject::MUST_READ_IF_MODIFIED
)
|| (io.readOpt() == IOobject::READ_IF_PRESENT && headerOk())
)
{
read();
}
else
{
List<char>::operator=(list);
}
}
Foam::decomposedBlockData::decomposedBlockData
(
const label comm,
const IOobject& io,
List<char>&& list,
const UPstream::commsTypes commsType
)
:
regIOobject(io),
commsType_(commsType),
comm_(comm)
{
// Temporary warning
if (io.readOpt() == IOobject::MUST_READ_IF_MODIFIED)
{
WarningInFunction
<< "decomposedBlockData " << name()
<< " constructed with IOobject::MUST_READ_IF_MODIFIED"
" but decomposedBlockData does not support automatic rereading."
<< endl;
}
List<char>::transfer(list);
if
(
(
io.readOpt() == IOobject::MUST_READ
|| io.readOpt() == IOobject::MUST_READ_IF_MODIFIED
)
|| (io.readOpt() == IOobject::READ_IF_PRESENT && headerOk())
)
{
read();
}
}
// * * * * * * * * * * * * * * * Members Functions * * * * * * * * * * * * * //
bool Foam::decomposedBlockData::readMasterHeader(IOobject& io, Istream& is)
{
if (debug)
{
Pout<< "decomposedBlockData::readMasterHeader:"
<< " stream:" << is.name() << endl;
}
// Master-only reading of header
is.fatalCheck("read(Istream&)");
List<char> data(is);
is.fatalCheck("read(Istream&) : reading entry");
UIListStream headerStream(data);
headerStream.name() = is.name();
return io.readHeader(headerStream);
}
void Foam::decomposedBlockData::writeHeader
bool Foam::decomposedBlockData::readBlockEntry
(
Ostream& os,
IOstreamOption streamOpt,
const word& objectType,
const string& note,
const fileName& location,
const word& objectName
Istream& is,
List<char>& charData
)
{
IOobject::writeBanner(os)
<< "FoamFile" << nl
<< '{' << nl
<< " version " << streamOpt.version() << ';' << nl
<< " format " << streamOpt.format() << ';' << nl
<< " arch " << foamVersion::buildArch << ';' << nl;
// Handle any of these:
if (Pstream::parRun())
// 0. NCHARS (...)
// 1. List<char> NCHARS (...)
// 2. processorN List<char> NCHARS (...) ;
is.fatalCheck(FUNCTION_NAME);
token tok(is);
is.fatalCheck(FUNCTION_NAME);
// Dictionary format has primitiveEntry keyword:
const bool isDictFormat = (tok.isWord() && !tok.isCompound());
if (!isDictFormat && tok.good())
{
os << " blocks " << Pstream::nProcs() << ';' << nl;
is.putBack(tok);
}
if (!note.empty())
charData.readList(is);
if (isDictFormat)
{
os << " note " << note << ';' << nl;
is.fatalCheck(FUNCTION_NAME);
is >> tok;
is.fatalCheck(FUNCTION_NAME);
// Swallow trailing ';'
if (tok.good() && !tok.isPunctuation(token::END_STATEMENT))
{
is.putBack(tok);
}
}
os << " class ";
if (objectType.empty())
{
// Empty type not allowed - use 'dictionary' fallback
os << "dictionary";
}
else
{
os << objectType;
}
os << ';' << nl;
if (!location.empty())
{
os << " location " << location << ';' << nl;
}
os << " object " << objectName << ';' << nl
<< '}' << nl;
IOobject::writeDivider(os) << nl;
return true;
}
Foam::autoPtr<Foam::ISstream> Foam::decomposedBlockData::readBlock
std::streamoff Foam::decomposedBlockData::writeBlockEntry
(
OSstream& os,
const label blocki,
const UList<char>& charData
)
{
// Offset to the beginning of this output
std::streamoff blockOffset = os.stdStream().tellp();
const word procName("processor" + Foam::name(blocki));
{
os << nl << "// " << procName << nl;
charData.writeList(os) << nl;
}
return blockOffset;
}
std::streamoff Foam::decomposedBlockData::writeBlockEntry
(
OSstream& os,
IOstreamOption streamOptData,
const regIOobject& io,
const label blocki,
const bool withLocalHeader
)
{
// String(s) from all data to write
string contentChars;
{
OStringStream os(streamOptData);
bool ok = true;
// Generate FoamFile header on master, without comment banner
if (withLocalHeader)
{
const bool old = IOobject::bannerEnabled(false);
ok = io.writeHeader(os);
IOobject::bannerEnabled(old);
}
// Write the data to the Ostream
ok = ok && io.writeData(os);
if (!ok)
{
return std::streamoff(-1);
}
contentChars = os.str();
}
// The character data
UList<char> charData
(
const_cast<char*>(contentChars.data()),
label(contentChars.size())
);
return decomposedBlockData::writeBlockEntry(os, blocki, charData);
}
Foam::autoPtr<Foam::ISstream>
Foam::decomposedBlockData::readBlock
(
const label blocki,
Istream& is,
ISstream& is,
IOobject& headerIO
)
{
@ -247,66 +237,64 @@ Foam::autoPtr<Foam::ISstream> Foam::decomposedBlockData::readBlock
<< endl;
}
is.fatalCheck("read(Istream&)");
// Extracted header information
IOstreamOption streamOptData;
unsigned labelWidth = is.labelByteSize();
unsigned scalarWidth = is.scalarByteSize();
autoPtr<ISstream> realIsPtr;
// Read master for header
List<char> data;
decomposedBlockData::readBlockEntry(is, data);
if (blocki == 0)
{
List<char> data(is);
is.fatalCheck("read(Istream&) : reading entry");
realIsPtr.reset(new IListStream(std::move(data)));
realIsPtr->name() = is.name();
// Read header
if (!headerIO.readHeader(*realIsPtr))
{
FatalIOErrorInFunction(*realIsPtr)
<< "problem while reading header for object "
<< is.name() << exit(FatalIOError);
// Read header from first block,
// advancing the stream position
if (!headerIO.readHeader(*realIsPtr))
{
FatalIOErrorInFunction(*realIsPtr)
<< "Problem while reading header for object "
<< is.name() << nl
<< exit(FatalIOError);
}
}
}
else
{
// Read master for header
List<char> data(is);
is.fatalCheck("read(Istream&) : reading entry");
IOstreamOption::versionNumber ver(IOstreamOption::currentVersion);
IOstreamOption::streamFormat fmt;
unsigned labelByteSize;
unsigned scalarByteSize;
{
// Read header from first block
UIListStream headerStream(data);
// Read header
if (!headerIO.readHeader(headerStream))
{
FatalIOErrorInFunction(headerStream)
<< "problem while reading header for object "
<< is.name() << exit(FatalIOError);
<< "Problem while reading header for object "
<< is.name() << nl
<< exit(FatalIOError);
}
ver = headerStream.version();
fmt = headerStream.format();
labelByteSize = headerStream.labelByteSize();
scalarByteSize = headerStream.scalarByteSize();
streamOptData = static_cast<IOstreamOption>(headerStream);
labelWidth = headerStream.labelByteSize();
scalarWidth = headerStream.scalarByteSize();
}
for (label i = 1; i < blocki+1; i++)
{
// Read and discard data, only retain the last one
is >> data;
is.fatalCheck("read(Istream&) : reading entry");
decomposedBlockData::readBlockEntry(is, data);
}
realIsPtr.reset(new IListStream(std::move(data)));
realIsPtr->name() = is.name();
// Apply master stream settings to realIsPtr
realIsPtr().format(fmt);
realIsPtr().version(ver);
realIsPtr().setLabelByteSize(labelByteSize);
realIsPtr().setScalarByteSize(scalarByteSize);
// Apply stream settings
realIsPtr().format(streamOptData.format());
realIsPtr().version(streamOptData.version());
realIsPtr().setLabelByteSize(labelWidth);
realIsPtr().setScalarByteSize(scalarWidth);
}
return realIsPtr;
@ -331,24 +319,28 @@ bool Foam::decomposedBlockData::readBlocks
bool ok = false;
if (UPstream::master(comm))
{
auto& is = *isPtr;
is.fatalCheck(FUNCTION_NAME);
// Read master data
decomposedBlockData::readBlockEntry(is, data);
}
if (commsType == UPstream::commsTypes::scheduled)
{
if (UPstream::master(comm))
{
Istream& is = *isPtr;
is.fatalCheck("read(Istream&)");
// Master data already read ...
auto& is = *isPtr;
is.fatalCheck(FUNCTION_NAME);
// Read master data
{
is >> data;
is.fatalCheck("read(Istream&) : reading entry");
}
// Read slave data
// Read and transmit slave data
for (const int proci : UPstream::subProcs(comm))
{
List<char> elems(is);
is.fatalCheck("read(Istream&) : reading entry");
List<char> elems;
decomposedBlockData::readBlockEntry(is, elems);
OPstream os
(
@ -387,20 +379,15 @@ bool Foam::decomposedBlockData::readBlocks
if (UPstream::master(comm))
{
Istream& is = *isPtr;
is.fatalCheck("read(Istream&)");
// Master data already read ...
auto& is = *isPtr;
is.fatalCheck(FUNCTION_NAME);
// Read master data
{
is >> data;
is.fatalCheck("read(Istream&) : reading entry");
}
// Read slave data
// Read and transmit slave data
for (const int proci : UPstream::subProcs(comm))
{
List<char> elems(is);
is.fatalCheck("read(Istream&) : reading entry");
List<char> elems;
decomposedBlockData::readBlockEntry(is, elems);
UOPstream os(proci, pBufs);
os << elems;
@ -440,39 +427,45 @@ Foam::autoPtr<Foam::ISstream> Foam::decomposedBlockData::readBlocks
}
bool ok = false;
List<char> data;
autoPtr<ISstream> realIsPtr;
if (UPstream::master(comm))
{
auto& is = *isPtr;
is.fatalCheck(FUNCTION_NAME);
// Read master data
decomposedBlockData::readBlockEntry(is, data);
realIsPtr.reset(new IListStream(std::move(data)));
realIsPtr->name() = fName;
{
// Read header from first block,
// advancing the stream position
if (!headerIO.readHeader(*realIsPtr))
{
FatalIOErrorInFunction(*realIsPtr)
<< "Problem while reading header for object "
<< is.name() << nl
<< exit(FatalIOError);
}
}
}
if (commsType == UPstream::commsTypes::scheduled)
{
if (UPstream::master(comm))
{
Istream& is = *isPtr;
is.fatalCheck("read(Istream&)");
// Master data already read ...
auto& is = *isPtr;
is.fatalCheck(FUNCTION_NAME);
// Read master data
{
is >> data;
is.fatalCheck("read(Istream&) : reading entry");
realIsPtr.reset(new IListStream(std::move(data)));
realIsPtr->name() = fName;
// Read header
if (!headerIO.readHeader(*realIsPtr))
{
FatalIOErrorInFunction(*realIsPtr)
<< "problem while reading header for object "
<< is.name() << exit(FatalIOError);
}
}
// Read slave data
// Read and transmit slave data
for (const int proci : UPstream::subProcs(comm))
{
is >> data;
is.fatalCheck("read(Istream&) : reading entry");
decomposedBlockData::readBlockEntry(is, data);
OPstream os
(
@ -514,31 +507,15 @@ Foam::autoPtr<Foam::ISstream> Foam::decomposedBlockData::readBlocks
if (UPstream::master(comm))
{
Istream& is = *isPtr;
is.fatalCheck("read(Istream&)");
// Master data already read ...
auto& is = *isPtr;
is.fatalCheck(FUNCTION_NAME);
// Read master data
{
is >> data;
is.fatalCheck("read(Istream&) : reading entry");
realIsPtr.reset(new IListStream(std::move(data)));
realIsPtr->name() = fName;
// Read header
if (!headerIO.readHeader(*realIsPtr))
{
FatalIOErrorInFunction(*realIsPtr)
<< "problem while reading header for object "
<< is.name() << exit(FatalIOError);
}
}
// Read slave data
// Read and transmit slave data
for (const int proci : UPstream::subProcs(comm))
{
List<char> elems(is);
is.fatalCheck("read(Istream&) : reading entry");
List<char> elems;
decomposedBlockData::readBlockEntry(is, elems);
UOPstream os(proci, pBufs);
os << elems;
@ -567,24 +544,24 @@ Foam::autoPtr<Foam::ISstream> Foam::decomposedBlockData::readBlocks
// Scatter master header info
int verValue;
int fmtValue;
unsigned labelByteSize;
unsigned scalarByteSize;
unsigned labelWidth;
unsigned scalarWidth;
if (UPstream::master(comm))
{
verValue = realIsPtr().version().canonical();
fmtValue = static_cast<int>(realIsPtr().format());
labelByteSize = realIsPtr().labelByteSize();
scalarByteSize = realIsPtr().scalarByteSize();
labelWidth = realIsPtr().labelByteSize();
scalarWidth = realIsPtr().scalarByteSize();
}
Pstream::scatter(verValue); //, Pstream::msgType(), comm);
Pstream::scatter(fmtValue); //, Pstream::msgType(), comm);
Pstream::scatter(labelByteSize); //, Pstream::msgType(), comm);
Pstream::scatter(scalarByteSize); //, Pstream::msgType(), comm);
Pstream::scatter(labelWidth); //, Pstream::msgType(), comm);
Pstream::scatter(scalarWidth); //, Pstream::msgType(), comm);
realIsPtr().version(IOstreamOption::versionNumber::canonical(verValue));
realIsPtr().format(IOstreamOption::streamFormat(fmtValue));
realIsPtr().setLabelByteSize(labelByteSize);
realIsPtr().setScalarByteSize(scalarByteSize);
realIsPtr().setLabelByteSize(labelWidth);
realIsPtr().setScalarByteSize(scalarWidth);
word name(headerIO.name());
Pstream::scatter(name, Pstream::msgType(), comm);
@ -656,8 +633,8 @@ void Foam::decomposedBlockData::gatherSlaveData
{
const label numProcs = UPstream::nProcs(comm);
sliceSizes.setSize(numProcs, 0);
sliceOffsets.setSize(numProcs+1, 0);
sliceSizes.resize(numProcs, 0);
sliceOffsets.resize(numProcs+1, 0);
int totalSize = 0;
label proci = startProc;
@ -741,8 +718,8 @@ bool Foam::decomposedBlockData::writeBlocks
(
const label comm,
autoPtr<OSstream>& osPtr,
List<std::streamoff>& start,
const UList<char>& data,
List<std::streamoff>& blockOffset,
const UList<char>& masterData,
const labelUList& recvSizes,
const PtrList<SubList<char>>& slaveData,
@ -755,7 +732,7 @@ bool Foam::decomposedBlockData::writeBlocks
{
Pout<< "decomposedBlockData::writeBlocks:"
<< " stream:" << (osPtr ? osPtr->name() : "invalid")
<< " data:" << data.size()
<< " data:" << masterData.size()
<< " (master only) slaveData:" << slaveData.size()
<< " commsType:" << Pstream::commsTypeNames[commsType] << endl;
}
@ -764,35 +741,43 @@ bool Foam::decomposedBlockData::writeBlocks
bool ok = true;
// Write master data
if (UPstream::master(comm))
{
blockOffset.resize(nProcs);
OSstream& os = *osPtr;
blockOffset[UPstream::masterNo()] =
decomposedBlockData::writeBlockEntry
(
os,
UPstream::masterNo(),
masterData
);
ok = os.good();
}
if (slaveData.size())
{
// Already have gathered the slave data. communicator only used to
// check who is the master
// Already have gathered the slave data.
if (UPstream::master(comm))
{
// Master data already written ...
OSstream& os = *osPtr;
start.setSize(nProcs);
// Write master data
{
os << nl << "// Processor" << UPstream::masterNo() << nl;
start[UPstream::masterNo()] = os.stdStream().tellp();
os << data;
}
// Write slaves
label slaveOffset = 0;
for (label proci = 1; proci < nProcs; ++proci)
{
os << nl << nl << "// Processor" << proci << nl;
start[proci] = os.stdStream().tellp();
os << slaveData[proci];
slaveOffset += recvSizes[proci];
blockOffset[proci] =
decomposedBlockData::writeBlockEntry
(
os,
proci,
slaveData[proci]
);
}
ok = os.good();
@ -802,21 +787,14 @@ bool Foam::decomposedBlockData::writeBlocks
{
if (UPstream::master(comm))
{
start.setSize(nProcs);
// Master data already written ...
OSstream& os = *osPtr;
// Write master data
{
os << nl << "// Processor" << UPstream::masterNo() << nl;
start[UPstream::masterNo()] = os.stdStream().tellp();
os << data;
}
// Write slaves
List<char> elems;
// Receive and write slaves
DynamicList<char> elems;
for (label proci = 1; proci < nProcs; ++proci)
{
elems.setSize(recvSizes[proci]);
elems.resize(recvSizes[proci]);
IPstream::read
(
UPstream::commsTypes::scheduled,
@ -827,9 +805,13 @@ bool Foam::decomposedBlockData::writeBlocks
comm
);
os << nl << nl << "// Processor" << proci << nl;
start[proci] = os.stdStream().tellp();
os << elems;
blockOffset[proci] =
decomposedBlockData::writeBlockEntry
(
os,
proci,
elems
);
}
ok = os.good();
@ -840,8 +822,8 @@ bool Foam::decomposedBlockData::writeBlocks
(
UPstream::commsTypes::scheduled,
UPstream::masterNo(),
data.cdata(),
data.size_bytes(),
masterData.cdata(),
masterData.size_bytes(),
Pstream::msgType(),
comm
);
@ -849,18 +831,7 @@ bool Foam::decomposedBlockData::writeBlocks
}
else
{
// Write master data
if (UPstream::master(comm))
{
start.setSize(nProcs);
OSstream& os = *osPtr;
os << nl << "// Processor" << UPstream::masterNo() << nl;
start[UPstream::masterNo()] = os.stdStream().tellp();
os << data;
}
// Master data already written ...
// Find out how many processor can be received into
// maxMasterFileBufferSize
@ -895,11 +866,11 @@ bool Foam::decomposedBlockData::writeBlocks
gatherSlaveData
(
comm,
data,
masterData,
recvSizes,
startProc, // startProc,
nSendProcs, // nProcs,
startProc, // startProc,
nSendProcs, // nProcs,
sliceOffsets,
recvData
@ -914,18 +885,22 @@ bool Foam::decomposedBlockData::writeBlocks
(
label proci = startProc;
proci < startProc+nSendProcs;
proci++
++proci
)
{
os << nl << nl << "// Processor" << proci << nl;
start[proci] = os.stdStream().tellp();
SubList<char> dataSlice
(
recvData,
sliceOffsets[proci+1]-sliceOffsets[proci],
sliceOffsets[proci]
);
os <<
SubList<char>
blockOffset[proci] =
decomposedBlockData::writeBlockEntry
(
recvData,
sliceOffsets[proci+1]-sliceOffsets[proci],
sliceOffsets[proci]
os,
proci,
dataSlice
);
}
}
@ -960,15 +935,12 @@ bool Foam::decomposedBlockData::read()
IOobject::readHeader(*isPtr);
}
List<char>& data = *this;
return readBlocks(comm_, isPtr, data, commsType_);
return readBlocks(comm_, isPtr, contentData_, commsType_);
}
bool Foam::decomposedBlockData::writeData(Ostream& os) const
{
const List<char>& data = *this;
IOobject io(*this);
IOstreamOption streamOpt(os);
@ -978,7 +950,7 @@ bool Foam::decomposedBlockData::writeData(Ostream& os) const
// Re-read my own data to find out the header information
if (Pstream::master(comm_))
{
UIListStream headerStream(data);
UIListStream headerStream(contentData_);
io.readHeader(headerStream);
verValue = headerStream.version().canonical();
@ -1008,7 +980,7 @@ bool Foam::decomposedBlockData::writeData(Ostream& os) const
decomposedBlockData::writeHeader
(
os,
streamOpt,
streamOpt, // streamOpt for data
io.headerClassName(),
io.note(),
masterLocation,
@ -1020,12 +992,12 @@ bool Foam::decomposedBlockData::writeData(Ostream& os) const
if (isA<OFstream>(os))
{
// Serial file output - can use writeRaw()
os.writeRaw(data.cdata(), data.size_bytes());
os.writeRaw(contentData_.cdata(), contentData_.size_bytes());
}
else
{
// Other cases are less fortunate, and no std::string_view
std::string str(data.cdata(), data.size_bytes());
std::string str(contentData_.cdata(), contentData_.size_bytes());
os.writeQuoted(str, false);
}
@ -1044,29 +1016,33 @@ bool Foam::decomposedBlockData::writeObject
const bool valid
) const
{
// Always write BINARY
streamOpt.format(IOstream::BINARY);
autoPtr<OSstream> osPtr;
if (UPstream::master(comm_))
{
// Note: always write binary. These are strings so readable anyway.
// They have already be tokenised on the sending side.
osPtr.reset(new OFstream(objectPath(), streamOpt));
IOobject::writeHeader(*osPtr);
osPtr.reset(new OFstream(objectPath(), IOstreamOption::BINARY));
decomposedBlockData::writeHeader
(
*osPtr,
streamOpt, // streamOpt for data
static_cast<const IOobject&>(*this)
);
}
labelList recvSizes;
gather(comm_, label(this->size_bytes()), recvSizes);
gather(comm_, label(contentData_.size_bytes()), recvSizes);
List<std::streamoff> start;
List<std::streamoff> blockOffsets;
PtrList<SubList<char>> slaveData; // dummy slave data
return writeBlocks
(
comm_,
osPtr,
start,
*this,
blockOffsets,
contentData_,
recvSizes,
slaveData,
commsType_
@ -1074,51 +1050,4 @@ bool Foam::decomposedBlockData::writeObject
}
Foam::label Foam::decomposedBlockData::numBlocks(const fileName& fName)
{
label nBlocks = 0;
IFstream is(fName);
is.fatalCheck(FUNCTION_NAME);
if (!is.good())
{
return nBlocks;
}
// FoamFile header
token firstToken(is);
if (is.good() && firstToken.isWord("FoamFile"))
{
dictionary headerDict(is);
is.version(headerDict.get<token>("version"));
is.format(headerDict.get<word>("format"));
// Obtain number of blocks directly
if (headerDict.readIfPresent("blocks", nBlocks))
{
return nBlocks;
}
}
// Fallback to brute force read of each data block
List<char> data;
while (is.good())
{
token sizeToken(is);
if (!sizeToken.isLabel())
{
return nBlocks;
}
is.putBack(sizeToken);
is >> data;
nBlocks++;
}
return nBlocks;
}
// ************************************************************************* //

161
src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.H
View File

@ -28,10 +28,56 @@ Class
Foam::decomposedBlockData
Description
decomposedBlockData is a List<char> with IO on the master processor only.
The decomposedBlockData comprise a \c List\<char\> for each output
processor, typically with IO on the master processor only.
For decomposedBlockData, we make a distinction between the container
description and the individual block contents.
The \b FoamFile header specifies the container characteristics and thus
has \c class = \c %decomposedBlockData and normally \c format = \c binary.
This description refers to the \em entire file container, not the
individual blocks.
Each processor block is simply a binary chunk of characters and the
first block also contains the header description for all of the blocks.
For example,
\verbatim
FoamFile
{
version 2.0;
format binary;
arch "LSB;label=32;scalar=64";
class decomposedBlockData;
location "constant/polyMesh";
object points;
}
// processor0
NCHARS
(FoamFile
{
version 2.0;
format ascii;
arch "LSB;label=32;scalar=64";
class vectorField;
location "constant/polyMesh";
object points;
}
...content...
)
// processor1
NCHARS
(...content...)
...
\endverbatim
SourceFiles
decomposedBlockData.C
decomposedBlockDataHeader.C
\*---------------------------------------------------------------------------*/
@ -53,9 +99,22 @@ namespace Foam
class decomposedBlockData
:
public regIOobject,
public List<char>
public regIOobject
{
// Private Functions
//- Helper: write content for FoamFile IOobject header
static void writeHeaderContent
(
Ostream& os,
IOstreamOption streamOptContainer,
const word& objectType,
const string& note,
const fileName& location,
const word& objectName
);
protected:
// Protected Data
@ -66,11 +125,14 @@ protected:
//- Communicator for all parallel comms
const label comm_;
//- The block content
List<char> contentData_;
// Protected Member Functions
//- Helper: determine number of processors whose recvSizes fits
// ito maxBufferSize
//- into maxBufferSize
static label calcNumProcs
(
const label comm,
@ -84,7 +146,7 @@ protected:
(
const label comm,
autoPtr<ISstream>& isPtr,
List<char>& data,
List<char>& contentChars,
const UPstream::commsTypes commsType
);
@ -105,30 +167,12 @@ public:
const UPstream::commsTypes = UPstream::commsTypes::scheduled
);
//- Construct given an IOobject and for READ_IF_MODIFIED a List<char>
decomposedBlockData
(
const label comm,
const IOobject& io,
const UList<char>& list,
const UPstream::commsTypes = UPstream::commsTypes::scheduled
);
//- Construct by transferring the List contents
decomposedBlockData
(
const label comm,
const IOobject& io,
List<char>&& list,
const UPstream::commsTypes = UPstream::commsTypes::scheduled
);
//- Destructor
virtual ~decomposedBlockData() = default;
// Member functions
// Member Functions
//- Read object
virtual bool read();
@ -147,30 +191,71 @@ public:
// Helpers
//- Read header. Call only on master.
static bool readMasterHeader(IOobject&, Istream&);
//- True if object type is a known collated type
static bool isCollatedType(const word& objectType);
//- True if object header class is a known collated type
static bool isCollatedType(const IOobject& io);
//- Read header as per IOobject with additional handling of
//- decomposedBlockData
static bool readHeader(IOobject& io, Istream& is);
//- Helper: write FoamFile IOobject header
static void writeHeader
(
Ostream& os,
IOstreamOption streamOpt,
IOstreamOption streamOptContainer,
const word& objectType,
const string& note,
const fileName& location,
const word& objectName
);
//- Helper: write FoamFile IOobject header
static void writeHeader
(
Ostream& os,
IOstreamOption streamOptData,
const IOobject& io
);
//- Helper: read block of (binary) character data
static bool readBlockEntry
(
Istream& is,
List<char>& charData
);
//- Helper: write block of (binary) character data
static std::streamoff writeBlockEntry
(
OSstream& os,
const label blocki,
const UList<char>& charData
);
//- Helper: write block of (binary) character data
// \return -1 on error
static std::streamoff writeBlockEntry
(
OSstream& os,
IOstreamOption streamOptData,
const regIOobject& io,
const label blocki,
const bool withLocalHeader
);
//- Read selected block (non-seeking) + header information
static autoPtr<ISstream> readBlock
(
const label blocki,
Istream& is,
ISstream& is,
IOobject& headerIO
);
//- Read master header information (into headerIO) and return
// data in stream. Note: isPtr is only valid on master.
//- data in stream. Note: isPtr is only valid on master.
static autoPtr<ISstream> readBlocks
(
const label comm,
@ -190,9 +275,11 @@ public:
labelList& datas
);
//- Helper: gather data from (subset of) slaves. Returns
// recvData : received data
// recvOffsets : offset in data. recvOffsets is nProcs+1
//- Helper: gather data from (subset of) slaves.
//
// Returns:
// - recvData : received data
// - recvOffsets : offset in data. recvOffsets is nProcs+1
static void gatherSlaveData
(
const label comm,
@ -206,13 +293,14 @@ public:
List<char>& recvData
);
//- Write *this. Ostream only valid on master. Returns starts of
// processor blocks
//- Write *this. Ostream only valid on master.
// Returns offsets of processor blocks in blockOffset
static bool writeBlocks
(
const label comm,
autoPtr<OSstream>& osPtr,
List<std::streamoff>& start,
List<std::streamoff>& blockOffset,
const UList<char>& masterData,
const labelUList& recvSizes,
@ -223,9 +311,6 @@ public:
const UPstream::commsTypes,
const bool syncReturnState = true
);
//- Detect number of blocks in a file
static label numBlocks(const fileName& fName);
};

217
src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockDataHeader.C Normal file
View File

@ -0,0 +1,217 @@
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2021 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
\*---------------------------------------------------------------------------*/
#include "decomposedBlockData.H"
#include "dictionary.H"
#include "foamVersion.H"
#include "objectRegistry.H"
#include "ListStream.H"
// * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
namespace Foam
{
inline void writeSpaces(Ostream& os, label nSpaces)
{
if (nSpaces < 1)
{
nSpaces = 1;
}
while (nSpaces--)
{
os.write(char(token::SPACE));
}
}
// Similar to writeEntry, but with fewer spaces
template<class T>
inline void writeHeaderEntry(Ostream& os, const word& key, const T& value)
{
os << indent << key;
writeSpaces(os, 12 - label(key.size()));
os << value << char(token::END_STATEMENT) << nl;
}
} // End namespace Foam
// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
void Foam::decomposedBlockData::writeHeaderContent
(
Ostream& os,
IOstreamOption streamOptContainer,
const word& objectType,
const string& note,
const fileName& location,
const word& objectName
)
{
// Standard header entries
writeHeaderEntry(os, "version", streamOptContainer.version());
writeHeaderEntry(os, "format", streamOptContainer.format());
writeHeaderEntry(os, "arch", foamVersion::buildArch);
if (!note.empty())
{
writeHeaderEntry(os, "note", note);
}
if (objectType.empty())
{
// Empty type not allowed - use 'dictionary' fallback
writeHeaderEntry(os, "class", word("dictionary"));
}
else
{
writeHeaderEntry(os, "class", objectType);
}
if (!location.empty())
{
writeHeaderEntry(os, "location", location);
}
writeHeaderEntry(os, "object", objectName);
}
// * * * * * * * * * * * * * * * Members Functions * * * * * * * * * * * * * //
bool Foam::decomposedBlockData::readHeader(IOobject& io, Istream& is)
{
dictionary headerDict;
// Read the regular "FoamFile" header
bool ok = io.readHeader(headerDict, is);
if (decomposedBlockData::isCollatedType(io))
{
// Quick information - extract from "data.class"
if (headerDict.readIfPresent("data.class", io.headerClassName()))
{
return ok;
}
{
// Master-only reading of header
List<char> charData;
decomposedBlockData::readBlockEntry(is, charData);
UIListStream headerStream(charData);
headerStream.name() = is.name();
ok = io.readHeader(headerStream);
}
}
return ok;
}
void Foam::decomposedBlockData::writeHeader
(
Ostream& os,
IOstreamOption streamOptContainer,
const word& objectType,
const string& note,
const fileName& location,
const word& objectName
)
{
if (IOobject::bannerEnabled())
{
IOobject::writeBanner(os);
}
os.beginBlock("FoamFile");
decomposedBlockData::writeHeaderContent
(
os,
streamOptContainer,
objectType,
note,
location,
objectName
);
os.endBlock();
if (IOobject::bannerEnabled())
{
IOobject::writeDivider(os) << nl;
}
}
void Foam::decomposedBlockData::writeHeader
(
Ostream& os,
IOstreamOption streamOptData,
const IOobject& io
)
{
if (IOobject::bannerEnabled())
{
IOobject::writeBanner(os);
}
os.beginBlock("FoamFile");
decomposedBlockData::writeHeaderContent
(
os,
static_cast<IOstreamOption>(os), // streamOpt container
decomposedBlockData::typeName, // class
io.note(),
(io.instance()/io.db().dbDir()/io.local()), // location
io.name()
);
{
writeHeaderEntry(os, "data.format", streamOptData.format());
writeHeaderEntry(os, "data.class", io.type());
}
// Meta-data (if any)
const dictionary* metaDataDict = io.findMetaData();
if (metaDataDict && !metaDataDict->empty())
{
metaDataDict->writeEntry("meta", os);
}
os.endBlock();
if (IOobject::bannerEnabled())
{
IOobject::writeDivider(os) << nl;
}
}
// ************************************************************************* //

26
src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.C
View File

@ -58,6 +58,7 @@ bool Foam::OFstreamCollator::writeFile
Pout<< "OFstreamCollator : Writing master " << masterData.size()
<< " bytes to " << fName
<< " using comm " << comm << endl;
if (slaveData.size())
{
Pout<< "OFstreamCollator : Slave data" << endl;
@ -85,10 +86,10 @@ bool Foam::OFstreamCollator::writeFile
decomposedBlockData::writeHeader
(
*osPtr,
streamOpt,
streamOpt, // streamOpt for container
objectType,
"", // note
fName, // location
"", // location (leave empty instead inaccurate)
fName.name() // object name
);
}
@ -106,12 +107,12 @@ bool Foam::OFstreamCollator::writeFile
// the master processor in order. However can be unstable
// for some mpi so default is non-blocking.
List<std::streamoff> start;
List<std::streamoff> blockOffset;
decomposedBlockData::writeBlocks
(
comm,
osPtr,
start,
blockOffset,
slice,
recvSizes,
slaveData,
@ -141,11 +142,8 @@ bool Foam::OFstreamCollator::writeFile
{
sum += recv;
}
// Use ostringstream to display long int (until writing these is
// supported)
std::ostringstream os;
os << sum;
Pout<< " (overall " << os.str() << ")";
// Use std::to_string to display long int
Pout<< " (overall " << std::to_string(sum) << ')';
}
Pout<< " to " << fName
<< " using comm " << comm << endl;
@ -355,10 +353,10 @@ bool Foam::OFstreamCollator::write
off_t totalSize = 0;
label maxLocalSize = 0;
{
for (label proci = 0; proci < recvSizes.size(); proci++)
for (const label recvSize : recvSizes)
{
totalSize += recvSizes[proci];
maxLocalSize = max(maxLocalSize, recvSizes[proci]);
totalSize += recvSize;
maxLocalSize = max(maxLocalSize, recvSize);
}
Pstream::scatter(totalSize, Pstream::msgType(), localComm_);
Pstream::scatter(maxLocalSize, Pstream::msgType(), localComm_);
@ -443,7 +441,7 @@ bool Foam::OFstreamCollator::write
(
UPstream::commsTypes::nonBlocking,
proci,
reinterpret_cast<char*>(slaveData[proci].data()),
slaveData[proci].data(),
slaveData[proci].size_bytes(),
Pstream::msgType(),
localComm_
@ -458,7 +456,7 @@ bool Foam::OFstreamCollator::write
(
UPstream::commsTypes::nonBlocking,
0,
reinterpret_cast<const char*>(slice.cdata()),
slice.cdata(),
slice.size_bytes(),
Pstream::msgType(),
localComm_

12
src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.H
View File

@ -71,10 +71,8 @@ class OFstreamCollator
{
// Private Class
class writeData
struct writeData
{
public:
const label comm_;
const word objectType_;
const fileName pathName_;
@ -105,18 +103,18 @@ class OFstreamCollator
append_(append)
{}
//- (approximate) size of master + any optional slave data
//- The (approximate) size of master + any optional slave data
off_t size() const
{
off_t sz = data_.size();
off_t totalSize = data_.size();
forAll(slaveData_, i)
{
if (slaveData_.set(i))
{
sz += slaveData_[i].size();
totalSize += slaveData_[i].size();
}
}
return sz;
return totalSize;
}
};

410
src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C
View File

@ -74,13 +74,13 @@ namespace fileOperations
}
// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * //
Foam::labelList Foam::fileOperations::collatedFileOperation::ioRanks()
{
labelList ioRanks;
string ioRanksString(getEnv("FOAM_IORANKS"));
string ioRanksString(Foam::getEnv("FOAM_IORANKS"));
if (!ioRanksString.empty())
{
IStringStream is(ioRanksString);
@ -91,6 +91,102 @@ Foam::labelList Foam::fileOperations::collatedFileOperation::ioRanks()
}
// * * * * * * * * * * * * Protected Member Functions * * * * * * * * * * * //
void Foam::fileOperations::collatedFileOperation::printBanner
(
const bool printRanks
) const
{
DetailInfo
<< "I/O : " << this->type();
if (maxThreadFileBufferSize == 0)
{
DetailInfo
<< " [unthreaded] (maxThreadFileBufferSize = 0)." << nl
<< " Writing may be slow for large file sizes."
<< endl;
}
else
{
DetailInfo
<< " [threaded] (maxThreadFileBufferSize = "
<< maxThreadFileBufferSize << ")." << nl
<< " Requires buffer large enough to collect all data"
" or thread support" << nl
<< " enabled in MPI. If MPI thread support cannot be"
" enabled, deactivate" << nl
<< " threading by setting maxThreadFileBufferSize"
" to 0 in" << nl
<< " OpenFOAM etc/controlDict" << endl;
}
if (printRanks)
{
// Information about the ranks
stringList ioRanks(Pstream::nProcs());
if (Pstream::master(comm_))
{
// Don't usually need the pid
// ioRanks[Pstream::myProcNo()] = hostName()+"."+name(pid());
ioRanks[Pstream::myProcNo()] = hostName();
}
Pstream::gatherList(ioRanks);
DynamicList<label> offsetMaster(Pstream::nProcs());
forAll(ioRanks, ranki)
{
if (!ioRanks[ranki].empty())
{
offsetMaster.append(ranki);
}
}
if (offsetMaster.size() > 1)
{
DetailInfo
<< "IO nodes:" << nl << '(' << nl;
offsetMaster.append(Pstream::nProcs());
for (label group = 1; group < offsetMaster.size(); ++group)
{
const label beg = offsetMaster[group-1];
const label end = offsetMaster[group];
DetailInfo
<< " (" << ioRanks[beg].c_str() << ' '
<< (end-beg) << ')' << nl;
}
DetailInfo
<< ')' << nl;
}
}
if
(
regIOobject::fileModificationChecking
== regIOobject::inotifyMaster
)
{
WarningInFunction
<< "Resetting fileModificationChecking to inotify" << endl;
}
if
(
regIOobject::fileModificationChecking
== regIOobject::timeStampMaster
)
{
WarningInFunction
<< "Resetting fileModificationChecking to timeStamp" << endl;
}
}
bool Foam::fileOperations::collatedFileOperation::isMasterRank
(
const label proci
@ -121,9 +217,9 @@ bool Foam::fileOperations::collatedFileOperation::appendObject
IOstreamOption streamOpt
) const
{
// Append to processors/ file
// Append to processorsNN/ file
label proci = detectProcessorPath(io.objectPath());
const label proci = detectProcessorPath(io.objectPath());
if (debug)
{
@ -132,59 +228,15 @@ bool Foam::fileOperations::collatedFileOperation::appendObject
<< " appending processor " << proci
<< " data to " << pathName << endl;
}
if (proci == -1)
{
FatalErrorInFunction
<< "Not a valid processor path " << pathName
<< "Invalid processor path: " << pathName
<< exit(FatalError);
}
const bool isMaster = isMasterRank(proci);
// Determine local rank (offset) if the pathName is a per-rank one
label localProci = proci;
{
fileName path, procDir, local;
procRangeType group;
label nProcs;
splitProcessorPath(pathName, path, procDir, local, group, nProcs);
// The local rank (offset)
if (!group.empty())
{
localProci = proci - group.start();
}
}
// Create string from all data to write
string buf;
{
OStringStream os(streamOpt);
if (isMaster)
{
if (!io.writeHeader(os))
{
return false;
}
}
// Write the data to the Ostream
if (!io.writeData(os))
{
return false;
}
if (isMaster)
{
IOobject::writeEndDivider(os);
}
buf = os.str();
}
// Note: cannot do append + compression. This is a limitation
// of ogzstream (or rather most compressed formats)
@ -204,26 +256,20 @@ bool Foam::fileOperations::collatedFileOperation::appendObject
if (isMaster)
{
decomposedBlockData::writeHeader
(
os,
static_cast<IOstreamOption>(os),
decomposedBlockData::typeName, // class
"", // note
pathName, // location
pathName.name() // object
);
decomposedBlockData::writeHeader(os, streamOpt, io);
}
// Write data
UList<char> slice
std::streamoff blockOffset = decomposedBlockData::writeBlockEntry
(
const_cast<char*>(buf.data()),
label(buf.size())
os,
streamOpt,
io,
proci,
// With FoamFile header on master?
isMaster
);
os << nl << "// Processor" << localProci << nl << slice << nl;
return os.good();
return (blockOffset >= 0) && os.good();
}
@ -252,81 +298,9 @@ Foam::fileOperations::collatedFileOperation::collatedFileOperation
nProcs_(Pstream::nProcs()),
ioRanks_(ioRanks())
{
verbose = (verbose && Foam::infoDetailLevel > 0);
if (verbose)
if (verbose && Foam::infoDetailLevel > 0)
{
DetailInfo
<< "I/O : " << typeName
<< " (maxThreadFileBufferSize " << maxThreadFileBufferSize
<< ')' << endl;
if (maxThreadFileBufferSize == 0)
{
DetailInfo
<< " Threading not activated "
"since maxThreadFileBufferSize = 0." << nl
<< " Writing may run slowly for large file sizes."
<< endl;
}
else
{
DetailInfo
<< " Threading activated "
"since maxThreadFileBufferSize > 0." << nl
<< " Requires large enough buffer to collect all data"
" or thread support " << nl
<< " enabled in MPI. If thread support cannot be "
"enabled, deactivate" << nl
<< " threading by setting maxThreadFileBufferSize "
"to 0 in" << nl
<< " OpenFOAM etc/controlDict"
<< endl;
}
if (ioRanks_.size())
{
// Print a bit of information
stringList ioRanks(Pstream::nProcs());
if (Pstream::master(comm_))
{
ioRanks[Pstream::myProcNo()] = hostName()+"."+name(pid());
}
Pstream::gatherList(ioRanks);
DetailInfo
<< " IO nodes:" << nl;
for (const string& ranks : ioRanks)
{
if (!ranks.empty())
{
DetailInfo
<< " " << ranks << nl;
}
}
}
if
(
regIOobject::fileModificationChecking
== regIOobject::inotifyMaster
)
{
WarningInFunction
<< "Resetting fileModificationChecking to inotify" << endl;
}
if
(
regIOobject::fileModificationChecking
== regIOobject::timeStampMaster
)
{
WarningInFunction
<< "Resetting fileModificationChecking to timeStamp" << endl;
}
this->printBanner(ioRanks_.size());
}
}
@ -345,56 +319,9 @@ Foam::fileOperations::collatedFileOperation::collatedFileOperation
nProcs_(Pstream::nProcs()),
ioRanks_(ioRanks)
{
verbose = (verbose && Foam::infoDetailLevel > 0);
if (verbose)
if (verbose && Foam::infoDetailLevel > 0)
{
DetailInfo
<< "I/O : " << typeName
<< " (maxThreadFileBufferSize " << maxThreadFileBufferSize
<< ')' << endl;
if (maxThreadFileBufferSize == 0)
{
DetailInfo
<< " Threading not activated "
"since maxThreadFileBufferSize = 0." << nl
<< " Writing may run slowly for large file sizes."
<< endl;
}
else
{
DetailInfo
<< " Threading activated "
"since maxThreadFileBufferSize > 0." << nl
<< " Requires large enough buffer to collect all data"
" or thread support " << nl
<< " enabled in MPI. If thread support cannot be "
"enabled, deactivate" << nl
<< " threading by setting maxThreadFileBufferSize "
"to 0 in the OpenFOAM etc/controlDict" << nl
<< endl;
}
if
(
regIOobject::fileModificationChecking
== regIOobject::inotifyMaster
)
{
WarningInFunction
<< "Resetting fileModificationChecking to inotify" << endl;
}
if
(
regIOobject::fileModificationChecking
== regIOobject::timeStampMaster
)
{
WarningInFunction
<< "Resetting fileModificationChecking to timeStamp" << endl;
}
this->printBanner(ioRanks_.size());
}
}
@ -473,23 +400,22 @@ bool Foam::fileOperations::collatedFileOperation::writeObject
valid
);
// If any of these fail, return (leave error handling to Ostream class)
if (!os.good())
{
return false;
}
if (!io.writeHeader(os))
{
return false;
}
// Write the data to the Ostream
if (!io.writeData(os))
{
return false;
}
IOobject::writeEndDivider(os);
// If any of these fail, return
// (leave error handling to Ostream class)
return true;
const bool ok =
(
os.good()
&& io.writeHeader(os)
&& io.writeData(os)
);
if (ok)
{
IOobject::writeEndDivider(os);
}
return ok;
}
else
{
@ -517,24 +443,22 @@ bool Foam::fileOperations::collatedFileOperation::writeObject
valid
);
// If any of these fail, return (leave error handling to Ostream
// class)
if (!os.good())
{
return false;
}
if (!io.writeHeader(os))
{
return false;
}
// Write the data to the Ostream
if (!io.writeData(os))
{
return false;
}
IOobject::writeEndDivider(os);
// If any of these fail, return
// (leave error handling to Ostream class)
return true;
const bool ok =
(
os.good()
&& io.writeHeader(os)
&& io.writeData(os)
);
if (ok)
{
IOobject::writeEndDivider(os);
}
return ok;
}
else if (!Pstream::parRun())
{
@ -553,7 +477,7 @@ bool Foam::fileOperations::collatedFileOperation::writeObject
{
// Re-check static maxThreadFileBufferSize variable to see
// if needs to use threading
bool useThread = (maxThreadFileBufferSize > 0);
const bool useThread = (maxThreadFileBufferSize > 0);
if (debug)
{
@ -576,27 +500,25 @@ bool Foam::fileOperations::collatedFileOperation::writeObject
useThread
);
// If any of these fail, return (leave error handling to Ostream
// class)
if (!os.good())
{
return false;
}
if (Pstream::master(comm_) && !io.writeHeader(os))
{
return false;
}
// Write the data to the Ostream
if (!io.writeData(os))
{
return false;
}
// If any of these fail, return
// (leave error handling to Ostream class)
bool ok = os.good();
if (Pstream::master(comm_))
{
IOobject::writeEndDivider(os);
// Suppress comment banner
const bool old = IOobject::bannerEnabled(false);
ok = ok && io.writeHeader(os);
IOobject::bannerEnabled(old);
}
return true;
ok = ok && io.writeData(os);
// No end divider for collated output
return ok;
}
}
}
@ -629,7 +551,7 @@ Foam::word Foam::fileOperations::collatedFileOperation::processorsDir
{
procDir +=
+ "_"
+ Foam::name(procs[0])
+ Foam::name(procs.first())
+ "-"
+ Foam::name(procs.last());
}
@ -649,19 +571,19 @@ Foam::word Foam::fileOperations::collatedFileOperation::processorsDir
// Find lowest io rank
label minProc = 0;
label maxProc = nProcs_-1;
forAll(ioRanks_, i)
for (const label ranki : ioRanks_)
{
if (ioRanks_[i] >= nProcs_)
if (ranki >= nProcs_)
{
break;
}
else if (ioRanks_[i] <= proci)
else if (ranki <= proci)
{
minProc = ioRanks_[i];
minProc = ranki;
}
else
{
maxProc = ioRanks_[i]-1;
maxProc = ranki-1;
break;
}
}

12
src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.H
View File

@ -65,7 +65,7 @@ class collatedFileOperation
{
protected:
// Protected data
// Protected Data
//- Any communicator allocated by me
const label myComm_;
@ -82,15 +82,19 @@ protected:
const labelList ioRanks_;
// Private Member Functions
// Protected Member Functions
//- Retrieve list of IO ranks from FOAM_IORANKS env variable
static labelList ioRanks();
//- Print banner information, optionally with io ranks
void printBanner(const bool printRanks = false) const;
//- Is proci master of communicator (in parallel) or master of
// the io ranks (non-parallel)
//- the io ranks (non-parallel)
bool isMasterRank(const label proci) const;
//- Append to processors/ file
//- Append to processorsNN/ file
bool appendObject
(
const regIOobject& io,

26
src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.C
View File

@ -6,6 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2017-2018 OpenFOAM Foundation
Copyright (C) 2021 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -137,31 +138,14 @@ Foam::fileOperations::hostCollatedFileOperation::hostCollatedFileOperation
UPstream::worldComm,
subRanks(Pstream::nProcs())
),
(Pstream::parRun() ? labelList(0) : ioRanks()), // processor dirs
(Pstream::parRun() ? labelList() : ioRanks()), // processor dirs
typeName,
verbose
false // verbose
)
{
verbose = (verbose && Foam::infoDetailLevel > 0);
if (verbose)
if (verbose && Foam::infoDetailLevel > 0)
{
// Print a bit of information
stringList ioRanks(Pstream::nProcs());
if (Pstream::master(comm_))
{
ioRanks[Pstream::myProcNo()] = hostName()+"."+name(pid());
}
Pstream::gatherList(ioRanks);
Info<< " IO nodes:" << nl;
for (const string& ranks : ioRanks)
{
if (!ranks.empty())
{
Info<< " " << ranks << nl;
}
}
this->printBanner(ioRanks_.size());
}
}

53
src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2017-2018 OpenFOAM Foundation
Copyright (C) 2019-2020 OpenCFD Ltd.
Copyright (C) 2019-2021 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -636,23 +636,20 @@ bool Foam::fileOperation::writeObject
OSstream& os = *osPtr;
// If any of these fail, return (leave error handling to Ostream class)
if (!os.good())
const bool ok =
(
os.good()
&& io.writeHeader(os)
&& io.writeData(os)
);
if (ok)
{
return false;
IOobject::writeEndDivider(os);
}
if (!io.writeHeader(os))
{
return false;
}
// Write the data to the Ostream
if (!io.writeData(os))
{
return false;
}
IOobject::writeEndDivider(os);
return ok;
}
return true;
}
@ -1087,29 +1084,15 @@ Foam::label Foam::fileOperation::nProcs
}
nProcs = maxProc+1;
if (nProcs == 0 && Foam::isDir(dir/processorsBaseDir))
{
fileName pointsFile
(
dir
/processorsBaseDir
/"constant"
/local
/polyMesh::meshSubDir
/"points"
);
WarningInFunction
<< "Defunct collated naming: " << processorsBaseDir << nl
<< "Manually rename with the decomposition number. Eg," << nl << nl
<< " mv processors processors16" << nl << nl
<< "...returning 1" << endl;
if (Foam::isFile(pointsFile))
{
nProcs = decomposedBlockData::numBlocks(pointsFile);
}
else
{
WarningInFunction << "Cannot read file " << pointsFile
<< " to determine the number of decompositions."
<< " Returning 1" << endl;
}
nProcs = 1;
}
}
Pstream::scatter(nProcs, Pstream::msgType(), comm_);

65
src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C
View File

@ -832,7 +832,7 @@ masterUncollatedFileOperationInitialise(int& argc, char**& argv)
if (argv[i] == s)
{
index = i;
setEnv("FOAM_IORANKS", argv[i+1], true);
Foam::setEnv("FOAM_IORANKS", argv[i+1], true);
break;
}
}
@ -1782,12 +1782,8 @@ bool Foam::fileOperations::masterUncollatedFileOperation::readHeader
if (is.good())
{
ok = io.readHeader(is);
if (io.headerClassName() == decomposedBlockData::typeName)
{
// Read the header inside the container (master data)
ok = decomposedBlockData::readMasterHeader(io, is);
}
// Regular header or from decomposed data
ok = decomposedBlockData::readHeader(io, is);
}
}
}
@ -1831,22 +1827,8 @@ bool Foam::fileOperations::masterUncollatedFileOperation::readHeader
if (is.good())
{
result[proci] = io.readHeader(is);
if
(
io.headerClassName()
== decomposedBlockData::typeName
)
{
// Read the header inside the container
// (master data)
result[proci] = decomposedBlockData::
readMasterHeader
(
io,
is
);
}
result[proci] =
decomposedBlockData::readHeader(io, is);
headerClassName[proci] = io.headerClassName();
note[proci] = io.note();
}
@ -1905,7 +1887,7 @@ Foam::fileOperations::masterUncollatedFileOperation::readStream
// have no file to read from. This will only happen when using
// normal writing since then the fName for the valid processors is
// processorDDD/<instance>/.. . In case of collocated writing
// the fName is already rewritten to processors/.
// the fName is already rewritten to processorsNN/.
isPtr.reset(new IFstream(fName));
@ -1914,11 +1896,9 @@ Foam::fileOperations::masterUncollatedFileOperation::readStream
// Read header data (on copy)
headerIO.readHeader(*isPtr);
if (headerIO.headerClassName() == decomposedBlockData::typeName)
{
isCollated = true;
}
else if (!Pstream::parRun())
isCollated = decomposedBlockData::isCollatedType(headerIO);
if (!isCollated && !Pstream::parRun())
{
// Short circuit: non-collated format. No parallel bits.
// Copy header and return.
@ -2206,25 +2186,20 @@ bool Foam::fileOperations::masterUncollatedFileOperation::writeObject
OSstream& os = *osPtr;
// If any of these fail, return (leave error handling to Ostream class)
if (!os.good())
const bool ok =
(
os.good()
&& io.writeHeader(os)
&& io.writeData(os)
);
if (ok)
{
return false;
IOobject::writeEndDivider(os);
}
if (!io.writeHeader(os))
{
return false;
}
// Write the data to the Ostream
if (!io.writeData(os))
{
return false;
}
IOobject::writeEndDivider(os);
return true;
return ok;
}

12
src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C
View File

@ -508,13 +508,8 @@ bool Foam::fileOperations::uncollatedFileOperation::readHeader
return false;
}
bool ok = io.readHeader(*isPtr);
if (io.headerClassName() == decomposedBlockData::typeName)
{
// Read the header inside the container (master data)
ok = decomposedBlockData::readMasterHeader(io, *isPtr);
}
// Regular header or from decomposed data
bool ok = decomposedBlockData::readHeader(io, *isPtr);
if (debug)
{
@ -570,8 +565,9 @@ Foam::fileOperations::uncollatedFileOperation::readStream
<< exit(FatalIOError);
}
if (io.headerClassName() != decomposedBlockData::typeName)
if (!decomposedBlockData::isCollatedType(io))
{
// Short circuit: non-collated format.
return isPtr;
}
else