zhyang-dev/openfoam
1
0
Fork 0
mirror of https://develop.openfoam.com/Development/openfoam.git synced 2025-11-28 03:28:01 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

ENH: additional low-level raw binary output for Ostream.

Browse Source 
- low-level beginRaw(), writeRaw(), endRaw() methods.
  These can be used to directly add '()' decorators for serial output
  or prepare/cleanup parallel buffers.
  Used, for example, when outputting indirect lists in binary to avoid.
This commit is contained in:
Mark Olesen
2017-10-20 10:26:55 +02:00
parent 3cbf399470
commit 74f667a85b
12 changed files with 370 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
applications/test/IndirectList/Test-IndirectList.C
View File

@ -3,7 +3,7 @@
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011 OpenFOAM Foundation
\\/ M anipulation |
\\/ M anipulation | Copyright (C) 2017 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -27,15 +27,18 @@ Description
#include "IndirectList.H"
#include "IOstreams.H"
#include "Fstream.H"
#include "ListOps.H"
#include "labelIndList.H"
#include "argList.H"
using namespace Foam;
template<class ListType>
void printInfo(const ListType& lst)
{
Info<< "addr: " << flatOutput(lst.addressing()) << nl
Info<< "full: " << flatOutput(lst.completeList()) << nl
<< "addr: " << flatOutput(lst.addressing()) << nl
<< "list: " << flatOutput(lst) << nl
<< endl;
}
@ -61,6 +64,15 @@ void testFind(const T& val, const ListType& lst)
int main(int argc, char *argv[])
{
argList::addOption
(
"binary",
"file",
"write lists in binary to specified file"
);
argList args(argc, argv);
List<label> completeList(20);
forAll(completeList, i)
@ -104,6 +116,55 @@ int main(int argc, char *argv[])
printInfo(idl2);
printInfo(idl3);
fileName binaryOutput;
if (args.optionReadIfPresent("binary", binaryOutput))
{
Info<<"Writing output to " << binaryOutput << endl;
OFstream os(binaryOutput, IOstream::BINARY);
os.writeEntry("idl1", idl1);
os.writeEntry("idl2", idl2);
os.writeEntry("idl3", idl3);
}
if (Pstream::parRun())
{
if (Pstream::master())
{
Pout<< "full: " << flatOutput(idl3.completeList()) << nl
<< "send: " << flatOutput(idl3) << endl;
for
(
int slave = Pstream::firstSlave();
slave <= Pstream::lastSlave();
++slave
)
{
OPstream toSlave(Pstream::commsTypes::scheduled, slave);
toSlave << idl3;
}
}
else
{
// From master
IPstream fromMaster
(
Pstream::commsTypes::scheduled,
Pstream::masterNo()
);
List<label> recv(fromMaster);
Pout<<"recv: " << flatOutput(recv) << endl;
}
// MPI barrier
bool barrier = true;
Pstream::scatter(barrier);
}
Info<< "End\n" << endl;
return 0;

16
src/OpenFOAM/containers/Lists/FixedList/FixedListIO.C
View File

@ -91,10 +91,10 @@ Foam::Ostream& Foam::FixedList<T, Size>::writeList
// Write size (so it is valid dictionary entry) and start delimiter
os << Size << token::BEGIN_BLOCK;
// Write contents
// Contents
os << L[0];
// Write end delimiter
// End delimiter
os << token::END_BLOCK;
}
else if
@ -103,31 +103,31 @@ Foam::Ostream& Foam::FixedList<T, Size>::writeList
|| (Size <= unsigned(shortListLen) && contiguous<T>())
)
{
// Write start delimiter
// Start delimiter
os << token::BEGIN_LIST;
// Write contents
// Contents
forAll(L, i)
{
if (i) os << token::SPACE;
os << L[i];
}
// Write end delimiter
// End delimiter
os << token::END_LIST;
}
else
{
// Write start delimiter
// Start delimiter
os << nl << token::BEGIN_LIST << nl;
// Write contents
// Contents
forAll(L, i)
{
os << L[i] << nl;
}
// Write end delimiter
// End delimiter
os << token::END_LIST << nl;
}
}

16
src/OpenFOAM/containers/Lists/List/ListIO.C
View File

@ -64,10 +64,10 @@ Foam::Istream& Foam::operator>>(Istream& is, List<T>& L)
}
else if (firstToken.isLabel())
{
const label s = firstToken.labelToken();
const label sz = firstToken.labelToken();
// Set list length to that read
L.setSize(s);
L.setSize(sz);
// Read list contents depending on data format
@ -76,11 +76,11 @@ Foam::Istream& Foam::operator>>(Istream& is, List<T>& L)
// Read beginning of contents
const char delimiter = is.readBeginList("List");
if (s)
if (sz)
{
if (delimiter == token::BEGIN_LIST)
{
for (label i=0; i<s; ++i)
for (label i=0; i<sz; ++i)
{
is >> L[i];
@ -103,7 +103,7 @@ Foam::Istream& Foam::operator>>(Istream& is, List<T>& L)
"reading the single entry"
);
for (label i=0; i<s; ++i)
for (label i=0; i<sz; ++i)
{
L[i] = element;
}
@ -115,11 +115,11 @@ Foam::Istream& Foam::operator>>(Istream& is, List<T>& L)
}
else
{
// contents are binary and contiguous
// Contents are binary and contiguous
if (s)
if (sz)
{
is.read(reinterpret_cast<char*>(L.data()), s*sizeof(T));
is.read(reinterpret_cast<char*>(L.data()), sz*sizeof(T));
is.fatalCheck
(

10
src/OpenFOAM/containers/Lists/PtrList/PtrListIO.C
View File

@ -48,19 +48,19 @@ void Foam::PtrList<T>::read(Istream& is, const INew& inewt)
if (firstToken.isLabel())
{
// Read size of list
const label s = firstToken.labelToken();
const label sz = firstToken.labelToken();
// Set list length to that read
setSize(s);
setSize(sz);
// Read beginning of contents
const char delimiter = is.readBeginList("PtrList");
if (s)
if (sz)
{
if (delimiter == token::BEGIN_LIST)
{
forAll(*this, i)
for (label i=0; i<sz; ++i)
{
set(i, inewt(is));
@ -82,7 +82,7 @@ void Foam::PtrList<T>::read(Istream& is, const INew& inewt)
"reading the single entry"
);
for (label i=1; i<s; ++i)
for (label i=1; i<sz; ++i)
{
set(i, tPtr->clone());
}

65
src/OpenFOAM/containers/Lists/UIndirectList/UIndirectListIO.C
View File

@ -39,14 +39,16 @@ Foam::Ostream& Foam::UIndirectList<T>::writeList
{
const UIndirectList<T>& L = *this;
const label sz = L.size();
// Write list contents depending on data format
if (os.format() == IOstream::ASCII || !contiguous<T>())
{
// Can the contents be considered 'uniform' (ie, identical)?
bool uniform = (L.size() > 1 && contiguous<T>());
bool uniform = (sz > 1 && contiguous<T>());
if (uniform)
{
forAll(L, i)
for (label i=1; i < sz; ++i)
{
if (L[i] != L[0])
{
@ -58,65 +60,72 @@ Foam::Ostream& Foam::UIndirectList<T>::writeList
if (uniform)
{
// Write size and start delimiter
os << L.size() << token::BEGIN_BLOCK;
// Size and start delimiter
os << sz << token::BEGIN_BLOCK;
// Write contents
// Contents
os << L[0];
// Write end delimiter
// End delimiter
os << token::END_BLOCK;
}
else if
(
L.size() <= 1 || !shortListLen
|| (L.size() <= shortListLen && contiguous<T>())
sz <= 1 || !shortListLen
|| (sz <= shortListLen && contiguous<T>())
)
{
// Write size and start delimiter
os << L.size() << token::BEGIN_LIST;
// Size and start delimiter
os << sz << token::BEGIN_LIST;
// Write contents
forAll(L, i)
// Contents
for (label i=0; i < sz; ++i)
{
if (i) os << token::SPACE;
os << L[i];
}
// Write end delimiter
// End delimiter
os << token::END_LIST;
}
else
{
// Write size and start delimiter
os << nl << L.size() << nl << token::BEGIN_LIST << nl;
// Size and start delimiter
os << nl << sz << nl << token::BEGIN_LIST << nl;
// Write contents
forAll(L, i)
// Contents
for (label i=0; i < sz; ++i)
{
os << L[i] << nl;
}
// Write end delimiter
// End delimiter
os << token::END_LIST << nl;
}
}
else
{
// Contents are binary and contiguous
os << nl << L.size() << nl;
os << nl << sz << nl;
if (L.size())
if (sz)
{
// This is annoying, and wasteful, but currently no alternative
List<T> lst = L();
// The TOTAL number of bytes to be written.
// - possibly add start delimiter
os.beginRaw(sz*sizeof(T));
// write(...) includes surrounding start/end delimiters
os.write
(
reinterpret_cast<const char*>(lst.cdata()),
lst.byteSize()
);
// Contents
for (label i=0; i < sz; ++i)
{
os.writeRaw
(
reinterpret_cast<const char*>(&(L[i])),
sizeof(T)
);
}
// End delimiter and/or cleanup.
os.endRaw();
}
}

72
src/OpenFOAM/containers/Lists/UList/UListIO.C
View File

@ -76,14 +76,16 @@ Foam::Ostream& Foam::UList<T>::writeList
{
const UList<T>& L = *this;
const label sz = L.size();
// Write list contents depending on data format
if (os.format() == IOstream::ASCII || !contiguous<T>())
{
// Can the contents be considered 'uniform' (ie, identical)?
bool uniform = (L.size() > 1 && contiguous<T>());
bool uniform = (sz > 1 && contiguous<T>());
if (uniform)
{
forAll(L, i)
for (label i=1; i < sz; ++i)
{
if (L[i] != L[0])
{
@ -95,58 +97,62 @@ Foam::Ostream& Foam::UList<T>::writeList
if (uniform)
{
// Write size and start delimiter
os << L.size() << token::BEGIN_BLOCK;
// Size and start delimiter
os << sz << token::BEGIN_BLOCK;
// Write contents
// Contents
os << L[0];
// Write end delimiter
// End delimiter
os << token::END_BLOCK;
}
else if
(
L.size() <= 1 || !shortListLen
|| (L.size() <= shortListLen && contiguous<T>())
sz <= 1 || !shortListLen
|| (sz <= shortListLen && contiguous<T>())
)
{
// Write size and start delimiter
os << L.size() << token::BEGIN_LIST;
// Size and start delimiter
os << sz << token::BEGIN_LIST;
// Write contents
forAll(L, i)
// Contents
for (label i=0; i < sz; ++i)
{
if (i) os << token::SPACE;
os << L[i];
}
// Write end delimiter
// End delimiter
os << token::END_LIST;
}
else
{
// Write size and start delimiter
os << nl << L.size() << nl << token::BEGIN_LIST << nl;
// Size and start delimiter
os << nl << sz << nl << token::BEGIN_LIST << nl;
// Write contents
forAll(L, i)
// Contents
for (label i=0; i < sz; ++i)
{
os << L[i] << nl;
}
// Write end delimiter
// End delimiter
os << token::END_LIST << nl;
}
}
else
{
// Contents are binary and contiguous
os << nl << L.size() << nl;
os << nl << sz << nl;
if (L.size())
if (sz)
{
// write(...) includes surrounding start/end delimiters
os.write(reinterpret_cast<const char*>(L.cdata()), L.byteSize());
os.write
(
reinterpret_cast<const char*>(L.cdata()),
L.byteSize()
);
}
}
@ -184,29 +190,29 @@ Foam::Istream& Foam::operator>>(Istream& is, UList<T>& L)
)
);
// Check list length
const label s = elems.size();
const label sz = elems.size();
if (s != L.size())
if (sz != L.size())
{
FatalIOErrorInFunction(is)
<< "incorrect length for UList. Read " << s
<< "incorrect length for UList. Read " << sz
<< " expected " << L.size()
<< exit(FatalIOError);
}
for (label i=0; i<s; ++i)
for (label i=0; i<sz; ++i)
{
L[i] = elems[i];
}
}
else if (firstToken.isLabel())
{
const label s = firstToken.labelToken();
const label sz = firstToken.labelToken();
// Set list length to that read
if (s != L.size())
if (sz != L.size())
{
FatalIOErrorInFunction(is)
<< "incorrect length for UList. Read " << s
<< "incorrect length for UList. Read " << sz
<< " expected " << L.size()
<< exit(FatalIOError);
}
@ -218,11 +224,11 @@ Foam::Istream& Foam::operator>>(Istream& is, UList<T>& L)
// Read beginning of contents
const char delimiter = is.readBeginList("List");
if (s)
if (sz)
{
if (delimiter == token::BEGIN_LIST)
{
for (label i=0; i<s; ++i)
for (label i=0; i<sz; ++i)
{
is >> L[i];
@ -245,7 +251,7 @@ Foam::Istream& Foam::operator>>(Istream& is, UList<T>& L)
"reading the single entry"
);
for (label i=0; i<s; ++i)
for (label i=0; i<sz; ++i)
{
L[i] = element;
}
@ -259,9 +265,9 @@ Foam::Istream& Foam::operator>>(Istream& is, UList<T>& L)
{
// contents are binary and contiguous
if (s)
if (sz)
{
is.read(reinterpret_cast<char*>(L.data()), s*sizeof(T));
is.read(reinterpret_cast<char*>(L.data()), sz*sizeof(T));
is.fatalCheck
(

18
src/OpenFOAM/containers/Lists/UPtrList/UPtrListIO.C
View File

@ -31,18 +31,20 @@ License
template<class T>
Foam::Ostream& Foam::operator<<(Ostream& os, const UPtrList<T>& L)
{
// Write size and start delimiter
os << nl << indent << L.size() << nl
<< indent << token::BEGIN_LIST << incrIndent;
const label sz = L.size();
// Write contents
forAll(L, i)
// Size and start delimiter
os << nl << indent << sz << nl
<< indent << token::BEGIN_LIST << incrIndent << nl;
// Contents
for (label i=0; i < sz; ++i)
{
os << nl << L[i];
os << L[i] << nl;
}
// Write end delimiter
os << nl << decrIndent << indent << token::END_LIST << nl;
// End delimiter
os << decrIndent << indent << token::END_LIST << nl;
os.check(FUNCTION_NAME);
return os;

45
src/OpenFOAM/db/IOstreams/IOstreams/Ostream.H
View File

@ -97,47 +97,62 @@ public:
// Write functions
//- Write next token to stream
virtual Ostream& write(const token&) = 0;
virtual Ostream& write(const token& t) = 0;
//- Write character
virtual Ostream& write(const char) = 0;
virtual Ostream& write(const char c) = 0;
//- Write character string
virtual Ostream& write(const char*) = 0;
virtual Ostream& write(const char* str) = 0;
//- Write word
virtual Ostream& write(const word&) = 0;
virtual Ostream& write(const word& str) = 0;
//- Write keyType
// A plain word is written unquoted.
// A regular expression is written as a quoted string.
virtual Ostream& write(const keyType&);
virtual Ostream& write(const keyType& kw);
//- Write string
virtual Ostream& write(const string&) = 0;
virtual Ostream& write(const string& str) = 0;
//- Write std::string surrounded by quotes.
// Optional write without quotes.
virtual Ostream& writeQuoted
(
const std::string&,
const std::string& str,
const bool quoted=true
) = 0;
//- Write int32_t
virtual Ostream& write(const int32_t) = 0;
virtual Ostream& write(const int32_t val) = 0;
//- Write int64_t
virtual Ostream& write(const int64_t) = 0;
virtual Ostream& write(const int64_t val) = 0;
//- Write floatScalar
virtual Ostream& write(const floatScalar) = 0;
virtual Ostream& write(const floatScalar val) = 0;
//- Write doubleScalar
virtual Ostream& write(const doubleScalar) = 0;
virtual Ostream& write(const doubleScalar val) = 0;
//- Write binary block
virtual Ostream& write(const char*, std::streamsize) = 0;
//- Write binary block.
virtual Ostream& write(const char* data, std::streamsize count) = 0;
//- Emit begin marker for low-level raw binary output.
// The count should indicate the number of bytes for subsequent
// writeRaw calls.
virtual Ostream& beginRaw(std::streamsize count) = 0;
//- Low-level raw binary output.
virtual Ostream& writeRaw
(
const char* data,
std::streamsize count
) = 0;
//- Emit end marker for low-level raw binary output.
virtual Ostream& endRaw() = 0;
//- Add indentation characters
virtual void indent() = 0;
@ -164,11 +179,11 @@ public:
void decrIndent();
//- Write the keyword followed by an appropriate indentation
virtual Ostream& writeKeyword(const keyType&);
virtual Ostream& writeKeyword(const keyType& kw);
//- Write begin block group with the given name
// Increments indentation, adds newline.
virtual Ostream& beginBlock(const keyType&);
virtual Ostream& beginBlock(const keyType& keyword);
//- Write begin block group without a name
// Increments indentation, adds newline.

101
src/OpenFOAM/db/IOstreams/Pstreams/UOPstream.C
View File

@ -3,7 +3,7 @@
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2017 OpenFOAM Foundation
\\/ M anipulation | Copyright (C) 2016 OpenCFD Ltd.
\\/ M anipulation | Copyright (C) 2016-2017 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -26,15 +26,42 @@ License
#include "UOPstream.H"
#include "int.H"
#include "token.H"
#include <cctype>
// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
template<class T>
inline void Foam::UOPstream::writeToBuffer(const T& t)
inline void Foam::UOPstream::prepareBuffer
(
const size_t count,
const size_t align
)
{
writeToBuffer(&t, sizeof(T), sizeof(T));
if (!count)
{
return;
}
// The current output position
label pos = sendBuf_.size();
if (align > 1)
{
// Align output position. Pads sendBuf_.size() - oldPos characters.
pos = align + ((pos - 1) & ~(align - 1));
}
// Extend buffer (as required)
sendBuf_.reserve(max(1000, label(pos + count)));
// Move to the aligned output position
sendBuf_.setSize(pos);
}
template<class T>
inline void Foam::UOPstream::writeToBuffer(const T& val)
{
writeToBuffer(&val, sizeof(T), sizeof(T));
}
@ -55,25 +82,26 @@ inline void Foam::UOPstream::writeToBuffer
const size_t align
)
{
if (!sendBuf_.capacity())
if (!count)
{
sendBuf_.setCapacity(1000);
return;
}
label alignedPos = sendBuf_.size();
prepareBuffer(count, align);
if (align > 1)
// The aligned output position
const label pos = sendBuf_.size();
// Extend the addressable range for direct pointer access
sendBuf_.setSize(pos + count);
char* const __restrict__ buf = (sendBuf_.begin() + pos);
const char* const __restrict__ input = reinterpret_cast<const char*>(data);
for (size_t i = 0; i < count; ++i)
{
// Align bufPosition. Pads sendBuf_.size() - oldPos characters.
alignedPos = align + ((sendBuf_.size() - 1) & ~(align - 1));
buf[i] = input[i];
}
// Extend if necessary
sendBuf_.setSize(alignedPos + count);
const char* dataPtr = reinterpret_cast<const char*>(data);
size_t i = count;
while (i--) sendBuf_[alignedPos++] = *dataPtr++;
}
@ -135,7 +163,7 @@ Foam::UOPstream::~UOPstream()
{
if
(
!UOPstream::write
!UOPstream::write
(
commsType_,
toProcNo_,
@ -298,6 +326,41 @@ Foam::Ostream& Foam::UOPstream::write
}
Foam::Ostream& Foam::UOPstream::beginRaw
(
const std::streamsize count
)
{
if (format() != BINARY)
{
FatalErrorInFunction
<< "stream format not binary"
<< Foam::abort(FatalError);
}
// Alignment = 8, as per write(const char*, streamsize)
prepareBuffer(count, 8);
return *this;
}
Foam::Ostream& Foam::UOPstream::writeRaw
(
const char* data,
const std::streamsize count
)
{
// No check for format() == BINARY since this is either done in the
// beginRaw() method, or the caller knows what they are doing.
// Previously aligned and sizes reserved via beginRaw()
writeToBuffer(data, count, 1);
return *this;
}
void Foam::UOPstream::print(Ostream& os) const
{
os << "Writing from processor " << toProcNo_

27
src/OpenFOAM/db/IOstreams/Pstreams/UOPstream.H
View File

@ -3,7 +3,7 @@
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2014 OpenFOAM Foundation
\\/ M anipulation |
\\/ M anipulation | Copyright (C) 2017 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -72,9 +72,12 @@ class UOPstream
// Private Member Functions
//- Write a T to the transfer buffer
//- Prepare buffer for count bytes of output at specified alignment.
inline void prepareBuffer(const size_t count, const size_t align);
//- Write data to the transfer buffer
template<class T>
inline void writeToBuffer(const T& t);
inline void writeToBuffer(const T& val);
//- Write a char to the transfer buffer
inline void writeToBuffer(const char& c);
@ -182,6 +185,24 @@ public:
//- Write binary block with 8-byte alignment.
Ostream& write(const char* data, const std::streamsize count);
//- Begin marker for low-level raw binary output.
// The count should indicate the number of bytes for subsequent
// writeRaw calls.
Ostream& beginRaw(const std::streamsize count);
//- Low-level raw binary output.
Ostream& writeRaw
(
const char* data,
const std::streamsize count
);
//- End marker for low-level raw binary output.
Ostream& endRaw()
{
return *this;
}
//- Add indentation characters
void indent()
{}

48
src/OpenFOAM/db/IOstreams/Sstreams/OSstream.C
View File

@ -210,7 +210,24 @@ Foam::Ostream& Foam::OSstream::write(const doubleScalar val)
}
Foam::Ostream& Foam::OSstream::write(const char* buf, std::streamsize count)
Foam::Ostream& Foam::OSstream::write
(
const char* data,
const std::streamsize count
)
{
beginRaw(count);
writeRaw(data, count);
endRaw();
return *this;
}
Foam::Ostream& Foam::OSstream::beginRaw
(
const std::streamsize count
)
{
if (format() != BINARY)
{
@ -220,8 +237,6 @@ Foam::Ostream& Foam::OSstream::write(const char* buf, std::streamsize count)
}
os_ << token::BEGIN_LIST;
os_.write(buf, count);
os_ << token::END_LIST;
setState(os_.rdstate());
@ -229,9 +244,34 @@ Foam::Ostream& Foam::OSstream::write(const char* buf, std::streamsize count)
}
Foam::Ostream& Foam::OSstream::writeRaw
(
const char* data,
std::streamsize count
)
{
// No check for format() == BINARY since this is either done in the
// beginRaw() method, or the caller knows what they are doing.
os_.write(data, count);
setState(os_.rdstate());
return *this;
}
Foam::Ostream& Foam::OSstream::endRaw()
{
os_ << token::END_LIST;
setState(os_.rdstate());
return *this;
}
void Foam::OSstream::indent()
{
for (unsigned short i = 0; i < indentLevel_*indentSize_; i++)
for (unsigned short i = 0; i < indentLevel_*indentSize_; ++i)
{
os_ << ' ';
}

21
src/OpenFOAM/db/IOstreams/Sstreams/OSstream.H
View File

@ -143,7 +143,26 @@ public:
virtual Ostream& write(const doubleScalar val);
//- Write binary block
virtual Ostream& write(const char* buf, std::streamsize count);
virtual Ostream& write
(
const char* data,
const std::streamsize count
);
//- Begin marker for low-level raw binary output.
// The count should indicate the number of bytes for subsequent
// writeRaw calls.
virtual Ostream& beginRaw(const std::streamsize count);
//- Low-level raw binary output.
virtual Ostream& writeRaw
(
const char* data,
const std::streamsize count
);
//- End marker for low-level raw binary output.
virtual Ostream& endRaw();
//- Add indentation characters
virtual void indent();