zhyang-dev/openfoam
1
0
Fork 0
mirror of https://develop.openfoam.com/Development/openfoam.git synced 2025-11-28 03:28:01 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

ENH: reduce some overhead in fileOperations

Browse Source 
- more consistent handling of file format (#1587)
This commit is contained in:
Mark Olesen
2020-02-13 12:37:18 +01:00
committed by Mark Olesen
parent 42299dca22
commit 8498e3f516
4 changed files with 170 additions and 338 deletions
Download Patch File Download Diff File Expand all files Collapse all files

184
src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.C
View File

@ -6,6 +6,7 @@
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2017-2018 OpenFOAM Foundation Copyright (C) 2017-2018 OpenFOAM Foundation
Copyright (C) 2020 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
@ -30,7 +31,6 @@ License
#include "IPstream.H" #include "IPstream.H"
#include "PstreamBuffers.H" #include "PstreamBuffers.H"
#include "Fstream.H" #include "Fstream.H"
#include "StringStream.H"
#include "dictionary.H" #include "dictionary.H"
#include "objectRegistry.H" #include "objectRegistry.H"
#include "SubList.H" #include "SubList.H"
@ -160,12 +160,6 @@ Foam::decomposedBlockData::decomposedBlockData
} }
// * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * * //
Foam::decomposedBlockData::~decomposedBlockData()
{}
// * * * * * * * * * * * * * * * Members Functions * * * * * * * * * * * * * // // * * * * * * * * * * * * * * * Members Functions * * * * * * * * * * * * * //
bool Foam::decomposedBlockData::readMasterHeader(IOobject& io, Istream& is) bool Foam::decomposedBlockData::readMasterHeader(IOobject& io, Istream& is)
@ -181,15 +175,11 @@ bool Foam::decomposedBlockData::readMasterHeader(IOobject& io, Istream& is)
List<char> data(is); List<char> data(is);
is.fatalCheck("read(Istream&) : reading entry"); is.fatalCheck("read(Istream&) : reading entry");
IListStream str
(
std::move(data),
IOstream::ASCII,
IOstream::currentVersion,
is.name()
);
return io.readHeader(str); UIListStream headerStream(data);
headerStream.name() = is.name();
return io.readHeader(headerStream);
} }
@ -254,24 +244,15 @@ Foam::autoPtr<Foam::ISstream> Foam::decomposedBlockData::readBlock
is.fatalCheck("read(Istream&)"); is.fatalCheck("read(Istream&)");
List<char> data;
autoPtr<ISstream> realIsPtr; autoPtr<ISstream> realIsPtr;
if (blocki == 0) if (blocki == 0)
{ {
is >> data; List<char> data(is);
is.fatalCheck("read(Istream&) : reading entry"); is.fatalCheck("read(Istream&) : reading entry");
realIsPtr.reset realIsPtr.reset(new IListStream(std::move(data)));
( realIsPtr->name() = is.name();
new IListStream
(
std::move(data),
IOstream::ASCII,
IOstream::currentVersion,
is.name()
)
);
// Read header // Read header
if (!headerIO.readHeader(realIsPtr())) if (!headerIO.readHeader(realIsPtr()))
@ -284,7 +265,7 @@ Foam::autoPtr<Foam::ISstream> Foam::decomposedBlockData::readBlock
else else
{ {
// Read master for header // Read master for header
is >> data; List<char> data(is);
is.fatalCheck("read(Istream&) : reading entry"); is.fatalCheck("read(Istream&) : reading entry");
IOstream::versionNumber ver(IOstream::currentVersion); IOstream::versionNumber ver(IOstream::currentVersion);
@ -292,13 +273,7 @@ Foam::autoPtr<Foam::ISstream> Foam::decomposedBlockData::readBlock
unsigned labelByteSize; unsigned labelByteSize;
unsigned scalarByteSize; unsigned scalarByteSize;
{ {
UIListStream headerStream UIListStream headerStream(data);
(
data,
IOstream::ASCII,
IOstream::currentVersion,
is.name()
);
// Read header // Read header
if (!headerIO.readHeader(headerStream)) if (!headerIO.readHeader(headerStream))
@ -315,20 +290,12 @@ Foam::autoPtr<Foam::ISstream> Foam::decomposedBlockData::readBlock
for (label i = 1; i < blocki+1; i++) for (label i = 1; i < blocki+1; i++)
{ {
// Read data, override old data // Read and discard data, only retain the last one
is >> data; is >> data;
is.fatalCheck("read(Istream&) : reading entry"); is.fatalCheck("read(Istream&) : reading entry");
} }
realIsPtr.reset realIsPtr.reset(new IListStream(std::move(data)));
( realIsPtr->name() = is.name();
new IListStream
(
std::move(data),
IOstream::ASCII,
IOstream::currentVersion,
is.name()
)
);
// Apply master stream settings to realIsPtr // Apply master stream settings to realIsPtr
realIsPtr().format(fmt); realIsPtr().format(fmt);
@ -336,6 +303,7 @@ Foam::autoPtr<Foam::ISstream> Foam::decomposedBlockData::readBlock
realIsPtr().setLabelByteSize(labelByteSize); realIsPtr().setLabelByteSize(labelByteSize);
realIsPtr().setScalarByteSize(scalarByteSize); realIsPtr().setScalarByteSize(scalarByteSize);
} }
return realIsPtr; return realIsPtr;
} }
@ -358,22 +326,6 @@ bool Foam::decomposedBlockData::readBlocks
bool ok = false; bool ok = false;
//// Scatter master header info
//string ver;
//unsigned labelByteSize;
//unsigned scalarByteSize;
//if (UPstream::master(comm))
//{
// ver = isPtr().version().str();
// labelByteSize = isPtr().labelByteSize();
// scalarByteSize = isPtr().scalarByteSize();
//}
//Pstream::scatter(ver); //, Pstream::msgType(), comm);
//Pstream::scatter(labelByteSize); //, Pstream::msgType(), comm);
//Pstream::scatter(scalarByteSize); //, Pstream::msgType(), comm);
if (commsType == UPstream::commsTypes::scheduled) if (commsType == UPstream::commsTypes::scheduled)
{ {
if (UPstream::master(comm)) if (UPstream::master(comm))
@ -509,17 +461,8 @@ Foam::autoPtr<Foam::ISstream> Foam::decomposedBlockData::readBlocks
is >> data; is >> data;
is.fatalCheck("read(Istream&) : reading entry"); is.fatalCheck("read(Istream&) : reading entry");
realIsPtr.reset realIsPtr.reset(new IListStream(std::move(data)));
( realIsPtr->name() = fName;
new IListStream
(
std::move(data),
IOstream::ASCII,
IOstream::currentVersion,
fName
)
);
// Read header // Read header
if (!headerIO.readHeader(realIsPtr())) if (!headerIO.readHeader(realIsPtr()))
@ -566,16 +509,8 @@ Foam::autoPtr<Foam::ISstream> Foam::decomposedBlockData::readBlocks
); );
is >> data; is >> data;
realIsPtr.reset realIsPtr.reset(new IListStream(std::move(data)));
( realIsPtr->name() = fName;
new IListStream
(
std::move(data),
IOstream::ASCII,
IOstream::currentVersion,
fName
)
);
} }
} }
else else
@ -597,16 +532,8 @@ Foam::autoPtr<Foam::ISstream> Foam::decomposedBlockData::readBlocks
is >> data; is >> data;
is.fatalCheck("read(Istream&) : reading entry"); is.fatalCheck("read(Istream&) : reading entry");
realIsPtr.reset realIsPtr.reset(new IListStream(std::move(data)));
( realIsPtr->name() = fName;
new IListStream
(
std::move(data),
IOstream::ASCII,
IOstream::currentVersion,
fName
)
);
// Read header // Read header
if (!headerIO.readHeader(realIsPtr())) if (!headerIO.readHeader(realIsPtr()))
@ -643,16 +570,8 @@ Foam::autoPtr<Foam::ISstream> Foam::decomposedBlockData::readBlocks
UIPstream is(UPstream::masterNo(), pBufs); UIPstream is(UPstream::masterNo(), pBufs);
is >> data; is >> data;
realIsPtr.reset realIsPtr.reset(new IListStream(std::move(data)));
( realIsPtr->name() = fName;
new IListStream
(
std::move(data),
IOstream::ASCII,
IOstream::currentVersion,
fName
)
);
} }
} }
@ -661,26 +580,24 @@ Foam::autoPtr<Foam::ISstream> Foam::decomposedBlockData::readBlocks
//- Set stream properties from realIsPtr on master //- Set stream properties from realIsPtr on master
// Scatter master header info // Scatter master header info
string ver; string versionString;
string format; label formatValue;
unsigned labelByteSize; unsigned labelByteSize;
unsigned scalarByteSize; unsigned scalarByteSize;
if (UPstream::master(comm)) if (UPstream::master(comm))
{ {
ver = realIsPtr().version().str(); versionString = realIsPtr().version().str();
OStringStream os; formatValue = static_cast<label>(realIsPtr().format());
os << realIsPtr().format();
format = os.str();
labelByteSize = realIsPtr().labelByteSize(); labelByteSize = realIsPtr().labelByteSize();
scalarByteSize = realIsPtr().scalarByteSize(); scalarByteSize = realIsPtr().scalarByteSize();
} }
Pstream::scatter(ver); //, Pstream::msgType(), comm); Pstream::scatter(versionString); //, Pstream::msgType(), comm);
Pstream::scatter(format); //, Pstream::msgType(), comm); Pstream::scatter(formatValue); //, Pstream::msgType(), comm);
Pstream::scatter(labelByteSize); //, Pstream::msgType(), comm); Pstream::scatter(labelByteSize); //, Pstream::msgType(), comm);
Pstream::scatter(scalarByteSize); //, Pstream::msgType(), comm); Pstream::scatter(scalarByteSize); //, Pstream::msgType(), comm);
realIsPtr().version(IOstream::versionNumber(ver)); realIsPtr().version(IOstream::versionNumber(versionString));
realIsPtr().format(format); realIsPtr().format(IOstream::streamFormat(formatValue));
realIsPtr().setLabelByteSize(labelByteSize); realIsPtr().setLabelByteSize(labelByteSize);
realIsPtr().setScalarByteSize(scalarByteSize); realIsPtr().setScalarByteSize(scalarByteSize);
@ -1072,30 +989,17 @@ bool Foam::decomposedBlockData::writeData(Ostream& os) const
// Re-read my own data to find out the header information // Re-read my own data to find out the header information
if (Pstream::master(comm_)) if (Pstream::master(comm_))
{ {
UIListStream is UIListStream headerStream(data);
( io.readHeader(headerStream);
data,
IOstream::ASCII,
IOstream::currentVersion,
name()
);
io.readHeader(is);
} }
// Scatter header information // Scatter header information
// version
string versionString(os.version().str()); string versionString(os.version().str());
Pstream::scatter(versionString, Pstream::msgType(), comm_); Pstream::scatter(versionString, Pstream::msgType(), comm_);
// stream label formatValue(os.format());
string formatString; Pstream::scatter(formatValue, Pstream::msgType(), comm_);
{
OStringStream os;
os << os.format();
formatString = os.str();
Pstream::scatter(formatString, Pstream::msgType(), comm_);
}
//word masterName(name()); //word masterName(name());
//Pstream::scatter(masterName, Pstream::msgType(), comm_); //Pstream::scatter(masterName, Pstream::msgType(), comm_);
@ -1114,7 +1018,7 @@ bool Foam::decomposedBlockData::writeData(Ostream& os) const
( (
os, os,
IOstream::versionNumber(versionString), IOstream::versionNumber(versionString),
IOstream::formatEnum(formatString), IOstream::streamFormat(formatValue),
io.headerClassName(), io.headerClassName(),
io.note(), io.note(),
masterLocation, masterLocation,
@ -1122,12 +1026,18 @@ bool Foam::decomposedBlockData::writeData(Ostream& os) const
); );
} }
string str // Write the character data
( if (isA<OFstream>(os))
reinterpret_cast<const char*>(data.cbegin()), {
data.byteSize() // Serial file output - can use writeRaw()
); os.writeRaw(data.cdata(), data.byteSize());
os.writeQuoted(str, false); }
else
{
// Other cases are less fortunate, and no std::string_view
std::string str(data.cdata(), data.byteSize());
os.writeQuoted(str, false);
}
if (!Pstream::master(comm_)) if (!Pstream::master(comm_))
{ {

10
src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.H
View File

@ -6,6 +6,7 @@
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2017-2018 OpenFOAM Foundation Copyright (C) 2017-2018 OpenFOAM Foundation
Copyright (C) 2020 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
@ -57,7 +58,7 @@ class decomposedBlockData
{ {
protected: protected:
// Protected data // Protected Data
//- Type to use for gather //- Type to use for gather
const UPstream::commsTypes commsType_; const UPstream::commsTypes commsType_;
@ -90,6 +91,7 @@ protected:
public: public:
//- Declare type-name, virtual type (with debug switch)
TypeName("decomposedBlockData"); TypeName("decomposedBlockData");
@ -123,7 +125,7 @@ public:
//- Destructor //- Destructor
virtual ~decomposedBlockData(); virtual ~decomposedBlockData() = default;
// Member functions // Member functions
@ -133,7 +135,7 @@ public:
//- Write separated content. Assumes content is the serialised data //- Write separated content. Assumes content is the serialised data
// and that the master data contains a header // and that the master data contains a header
virtual bool writeData(Ostream&) const; virtual bool writeData(Ostream& os) const;
//- Write using stream options //- Write using stream options
virtual bool writeObject virtual bool writeObject
@ -224,7 +226,7 @@ public:
); );
//- Detect number of blocks in a file //- Detect number of blocks in a file
static label numBlocks(const fileName&); static label numBlocks(const fileName& fName);
}; };

301
src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C
View File

@ -494,93 +494,71 @@ bool Foam::fileOperations::masterUncollatedFileOperation::uniformFile
void Foam::fileOperations::masterUncollatedFileOperation::readAndSend void Foam::fileOperations::masterUncollatedFileOperation::readAndSend
( (
const fileName& filePath, const fileName& filePath,
const IOstream::compressionType cmp,
const labelUList& procs, const labelUList& procs,
PstreamBuffers& pBufs PstreamBuffers& pBufs
) )
{ {
if (cmp == IOstream::compressionType::COMPRESSED) IFstream ifs(filePath, IOstream::streamFormat::BINARY);
if (!ifs.good())
{ {
if (debug) FatalIOErrorInFunction(filePath)
{ << "Cannot open file " << filePath
Pout<< "masterUncollatedFileOperation::readAndSend :" << exit(FatalIOError);
<< " Opening compressed " << filePath << endl;
}
IFstream is(filePath, IOstream::streamFormat::BINARY);
if (!is.good())
{
FatalIOErrorInFunction(filePath) << "Cannot open file " << filePath
<< exit(FatalIOError);
}
std::ostringstream stringStr;
stringStr << is.stdStream().rdbuf();
string buf(stringStr.str());
forAll(procs, i)
{
UOPstream os(procs[i], pBufs);
os.write(&buf[0], buf.size());
}
} }
else
if (debug)
{ {
off_t count(Foam::fileSize(filePath)); Pout<< "masterUncollatedFileOperation::readAndSend :"
IFstream is(filePath, IOstream::streamFormat::BINARY); << " compressed:" << bool(ifs.compression()) << " "
<< filePath << endl;
}
if (!is.good()) if (ifs.compression() == IOstream::compressionType::COMPRESSED)
{
// Could use Foam::fileSize, estimate uncompressed size (eg, 2x)
// and then string reserve followed by string assign...
// Uncompress and read file contents into a character buffer
const std::string buf
(
std::istreambuf_iterator<char>(ifs.stdStream()),
std::istreambuf_iterator<char>()
);
for (const label proci : procs)
{ {
FatalIOErrorInFunction(filePath) << "Cannot open file " << filePath UOPstream os(proci, pBufs);
<< exit(FatalIOError); os.write(buf.data(), buf.length());
} }
if (debug) if (debug)
{ {
Pout<< "masterUncollatedFileOperation::readStream :" Pout<< "masterUncollatedFileOperation::readStream :"
<< " From " << filePath << " reading " << label(count) << " From " << filePath << " sent " << buf.size()
<< " bytes" << endl; << " bytes" << endl;
} }
List<char> buf(static_cast<label>(count));
is.stdStream().read(buf.begin(), count);
forAll(procs, i)
{
UOPstream os(procs[i], pBufs);
os.write(buf.begin(), count);
}
}
}
void Foam::fileOperations::masterUncollatedFileOperation::readAndSend
(
const fileName& fName,
const labelUList& procs,
PstreamBuffers& pBufs
)
{
if (Foam::exists(fName+".gz", false))
{
readAndSend
(
fName,
IOstream::compressionType::COMPRESSED,
procs,
pBufs
);
} }
else else
{ {
readAndSend const off_t count(Foam::fileSize(filePath));
(
fName, // Read file contents into a character buffer
IOstream::compressionType::UNCOMPRESSED, List<char> buf(static_cast<label>(count));
procs, ifs.stdStream().read(buf.data(), count);
pBufs
); for (const label proci : procs)
{
UOPstream os(proci, pBufs);
os.write(buf.cdata(), count);
}
if (debug)
{
Pout<< "masterUncollatedFileOperation::readStream :"
<< " From " << filePath << " sent " << buf.size()
<< " bytes" << endl;
}
} }
} }
@ -656,18 +634,16 @@ Foam::fileOperations::masterUncollatedFileOperation::read
<< exit(FatalIOError); << exit(FatalIOError);
} }
autoPtr<IFstream> ifsPtr(new IFstream(filePaths[0])); // Open master
isPtr.reset(new IFstream(filePaths[0]));
// Read header // Read header
if (!io.readHeader(ifsPtr())) if (!io.readHeader(isPtr()))
{ {
FatalIOErrorInFunction(ifsPtr()) FatalIOErrorInFunction(isPtr())
<< "problem while reading header for object " << "problem while reading header for object "
<< io.name() << exit(FatalIOError); << io.name() << exit(FatalIOError);
} }
// Open master (steal from ifsPtr)
isPtr.reset(ifsPtr.ptr());
} }
// Read slave files // Read slave files
@ -715,7 +691,7 @@ Foam::fileOperations::masterUncollatedFileOperation::read
List<char> buf(recvSizes[Pstream::masterNo()]); List<char> buf(recvSizes[Pstream::masterNo()]);
if (recvSizes[Pstream::masterNo()] > 0) if (recvSizes[Pstream::masterNo()] > 0)
{ {
is.read(buf.begin(), recvSizes[Pstream::masterNo()]); is.read(buf.data(), recvSizes[Pstream::masterNo()]);
} }
if (debug) if (debug)
@ -723,17 +699,15 @@ Foam::fileOperations::masterUncollatedFileOperation::read
Pout<< "masterUncollatedFileOperation::readStream :" Pout<< "masterUncollatedFileOperation::readStream :"
<< " Done reading " << buf.size() << " bytes" << endl; << " Done reading " << buf.size() << " bytes" << endl;
} }
const fileName& fName = filePaths[Pstream::myProcNo(comm)];
isPtr.reset // A local character buffer copy of the Pstream contents.
( // Construct with same parameters (ASCII, current version)
new IListStream // as the IFstream so that it has the same characteristics.
(
std::move(buf), isPtr.reset(new IListStream(std::move(buf)));
IOstream::BINARY,
IOstream::currentVersion, // With the proper file name
fName isPtr->name() = filePaths[Pstream::myProcNo(comm)];
)
);
if (!io.readHeader(isPtr())) if (!io.readHeader(isPtr()))
{ {
@ -1424,6 +1398,7 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::dirPath
{ {
// Retest all processors separately since some processors might // Retest all processors separately since some processors might
// have the file and some not (e.g. lagrangian data) // have the file and some not (e.g. lagrangian data)
objPath = masterOp<fileName, fileOrNullOp> objPath = masterOp<fileName, fileOrNullOp>
( (
io.objectPath(), io.objectPath(),
@ -1721,7 +1696,7 @@ Foam::fileOperations::masterUncollatedFileOperation::readObjects
{ {
// Avoid fileOperation::readObjects from triggering parallel ops // Avoid fileOperation::readObjects from triggering parallel ops
// (through call to filePath which triggers parallel ) // (through call to filePath which triggers parallel )
bool oldParRun = UPstream::parRun(); const bool oldParRun = UPstream::parRun();
UPstream::parRun() = false; UPstream::parRun() = false;
//- Use non-time searching version //- Use non-time searching version
@ -1872,8 +1847,8 @@ bool Foam::fileOperations::masterUncollatedFileOperation::readHeader
== decomposedBlockData::typeName == decomposedBlockData::typeName
) )
{ {
// Read the header inside the container (master // Read the header inside the container
// data) // (master data)
result[proci] = decomposedBlockData:: result[proci] = decomposedBlockData::
readMasterHeader readMasterHeader
( (
@ -2003,7 +1978,6 @@ Foam::fileOperations::masterUncollatedFileOperation::readStream
); );
List<char> data;
if (!Pstream::parRun()) if (!Pstream::parRun())
{ {
// Analyse the objectpath to find out the processor we're trying // Analyse the objectpath to find out the processor we're trying
@ -2038,20 +2012,6 @@ Foam::fileOperations::masterUncollatedFileOperation::readStream
} }
else else
{ {
// Scatter master header info
//string versionString;
//string formatString;
//if (isPtr.valid())
//{
// versionString = isPtr().version().str();
// OStringStream os;
// os << isPtr().format();
// formatString = (os.str());
//}
//
//Pstream::scatter(versionString); //, Pstream::msgType(), comm);
//Pstream::scatter(formatString); //, Pstream::msgType(), comm);
// Get size of file // Get size of file
off_t sz = Foam::fileSize(fName); off_t sz = Foam::fileSize(fName);
bool bigSize = sz > off_t(maxMasterFileBufferSize); bool bigSize = sz > off_t(maxMasterFileBufferSize);
@ -2131,7 +2091,7 @@ Foam::fileOperations::masterUncollatedFileOperation::readStream
Pstream::gatherList(procValid, Pstream::msgType(), comm_); Pstream::gatherList(procValid, Pstream::msgType(), comm_);
// Uniform in local comm // Uniform in local comm
bool uniform = uniformFile(filePaths); const bool uniform = uniformFile(filePaths);
return read(io, comm_, uniform, filePaths, procValid); return read(io, comm_, uniform, filePaths, procValid);
} }
@ -2161,7 +2121,7 @@ bool Foam::fileOperations::masterUncollatedFileOperation::read
if (Pstream::master()) // comm_)) if (Pstream::master()) // comm_))
{ {
// Do master-only reading always. // Do master-only reading always.
bool oldParRun = UPstream::parRun(); const bool oldParRun = UPstream::parRun();
UPstream::parRun() = false; UPstream::parRun() = false;
ok = io.readData(io.readStream(typeName)); ok = io.readData(io.readStream(typeName));
@ -2307,7 +2267,7 @@ Foam::instantList Foam::fileOperations::masterUncollatedFileOperation::findTimes
if (Pstream::master()) // comm_)) if (Pstream::master()) // comm_))
{ {
// Do master-only reading always. // Do master-only reading always.
bool oldParRun = UPstream::parRun(); const bool oldParRun = UPstream::parRun();
UPstream::parRun() = false; UPstream::parRun() = false;
times = fileOperation::findTimes(directory, constantName); times = fileOperation::findTimes(directory, constantName);
UPstream::parRun() = oldParRun; UPstream::parRun() = oldParRun;
@ -2343,6 +2303,8 @@ void Foam::fileOperations::masterUncollatedFileOperation::setTime
} }
// Mutable access to instantList for modification and sorting // Mutable access to instantList for modification and sorting
// - cannot use auto type deduction here
HashPtrTable<instantList>::iterator iter = times_.find(tm.path()); HashPtrTable<instantList>::iterator iter = times_.find(tm.path());
if (iter.found()) if (iter.found())
@ -2351,48 +2313,40 @@ void Foam::fileOperations::masterUncollatedFileOperation::setTime
const instant timeNow(tm.value(), tm.timeName()); const instant timeNow(tm.value(), tm.timeName());
if (times.size() > 0 && times[0].name() == tm.constant()) // Exclude constant when checking and sorting
{ const label skipConst =
// Exclude constant (
SubList<instant> realTimes(times, times.size()-1, 1); (!times.empty() && times[0].name() == tm.constant())
if ? 1
: 0
);
if
(
findSortedIndex
( (
findSortedIndex SubList<instant>(times, times.size()-skipConst, skipConst),
( timeNow
SubList<instant>(times, times.size()-1, 1),
timeNow
)
== -1
) )
{ == -1
if (debug) )
{
Pout<< "masterUncollatedFileOperation::setTime :"
<< " Caching time " << tm.timeName()
<< " for case:" << tm.path() << endl;
}
times.append(timeNow);
SubList<instant> realTimes(times, times.size()-1, 1);
Foam::stableSort(realTimes);
}
}
else
{ {
if (findSortedIndex(times, timeNow) == -1) if (debug)
{ {
if (debug) Pout<< "masterUncollatedFileOperation::setTime :"
{ << " Caching time " << tm.timeName()
Pout<< "masterUncollatedFileOperation::setTime :" << " for case:" << tm.path() << endl;
<< " Caching time " << tm.timeName()
<< " for case:" << tm.path() << endl;
}
times.append(timeNow);
Foam::stableSort(times);
} }
times.append(timeNow);
SubList<instant> realTimes
(
times, times.size()-skipConst, skipConst
);
Foam::stableSort(realTimes);
} }
} }
fileOperation::setTime(tm); fileOperation::setTime(tm);
} }
@ -2403,6 +2357,8 @@ Foam::fileOperations::masterUncollatedFileOperation::NewIFstream
const fileName& filePath const fileName& filePath
) const ) const
{ {
autoPtr<ISstream> isPtr;
if (Pstream::parRun()) if (Pstream::parRun())
{ {
// Insert logic of filePath. We assume that if a file is absolute // Insert logic of filePath. We assume that if a file is absolute
@ -2431,25 +2387,12 @@ Foam::fileOperations::masterUncollatedFileOperation::NewIFstream
<< " Opening global file " << filePath << endl; << " Opening global file " << filePath << endl;
} }
IOstream::compressionType cmp readAndSend
( (
Foam::exists(filePath+".gz", false) filePath,
? IOstream::compressionType::COMPRESSED identity(Pstream::nProcs(Pstream::worldComm)-1, 1),
: IOstream::compressionType::UNCOMPRESSED pBufs
); );
labelList procs(Pstream::nProcs(Pstream::worldComm)-1);
for
(
label proci = 1;
proci < Pstream::nProcs(Pstream::worldComm);
proci++
)
{
procs[proci-1] = proci;
}
readAndSend(filePath, cmp, procs, pBufs);
} }
else else
{ {
@ -2460,17 +2403,9 @@ Foam::fileOperations::masterUncollatedFileOperation::NewIFstream
proci++ proci++
) )
{ {
IOstream::compressionType cmp
(
Foam::exists(filePaths[proci]+".gz", false)
? IOstream::compressionType::COMPRESSED
: IOstream::compressionType::UNCOMPRESSED
);
readAndSend readAndSend
( (
filePaths[proci], filePaths[proci],
cmp,
labelList(1, proci), labelList(1, proci),
pBufs pBufs
); );
@ -2485,10 +2420,7 @@ Foam::fileOperations::masterUncollatedFileOperation::NewIFstream
if (Pstream::master(Pstream::worldComm)) if (Pstream::master(Pstream::worldComm))
{ {
// Read myself // Read myself
return autoPtr<ISstream> isPtr.reset(new IFstream(filePaths[Pstream::masterNo()]));
(
new IFstream(filePaths[Pstream::masterNo()])
);
} }
else else
{ {
@ -2502,7 +2434,7 @@ Foam::fileOperations::masterUncollatedFileOperation::NewIFstream
UIPstream is(Pstream::masterNo(), pBufs); UIPstream is(Pstream::masterNo(), pBufs);
List<char> buf(recvSizes[Pstream::masterNo()]); List<char> buf(recvSizes[Pstream::masterNo()]);
is.read(buf.begin(), buf.size()); is.read(buf.data(), buf.size());
if (debug) if (debug)
{ {
@ -2510,26 +2442,23 @@ Foam::fileOperations::masterUncollatedFileOperation::NewIFstream
<< " Done reading " << buf.size() << " bytes" << endl; << " Done reading " << buf.size() << " bytes" << endl;
} }
// Note: IPstream is not an IStream so use a IStringStream to // A local character buffer copy of the Pstream contents.
// convert the buffer. Note that we construct with a string // Construct with same parameters (ASCII, current version)
// so it holds a copy of the buffer. // as the IFstream so that it has the same characteristics.
return autoPtr<ISstream>
( isPtr.reset(new IListStream(std::move(buf)));
new IListStream
( // With the proper file name
std::move(buf), isPtr->name() = filePath;
IOstream::BINARY,
IOstream::currentVersion,
filePath
)
);
} }
} }
else else
{ {
// Read myself // Read myself
return autoPtr<ISstream>(new IFstream(filePath)); isPtr.reset(new IFstream(filePath));
} }
return isPtr;
} }

13
src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.H
View File

@ -448,20 +448,11 @@ protected:
const word& instancePath const word& instancePath
) const; ) const;
//- Read file contents and send to processors //- Read file contents and send to processors.
// Handles compressed or uncompressed files
static void readAndSend static void readAndSend
( (
const fileName& filePath, const fileName& filePath,
const IOstream::compressionType cmp,
const labelUList& procs,
PstreamBuffers& pBufs
);
//- Detect file (possibly compressed), read file contents and send
// to processors
static void readAndSend
(
const fileName& fName,
const labelUList& procs, const labelUList& procs,
PstreamBuffers& pBufs PstreamBuffers& pBufs
); );