From ef9bb4ae1654949a19db7edf40c2fa050d540d66 Mon Sep 17 00:00:00 2001 From: Mark Olesen Date: Mon, 29 Jul 2019 16:01:34 +0200 Subject: [PATCH] ENH: add low-level readRawLabels, readRawScalars (#1378) - these use the additional byte-size checks in IOstream to handle native vs non-native sizes --- applications/test/limits/Test-limits.C | 63 ++++++++---- .../test/primitives/Test-primitives.C | 69 +++++++++++++- .../db/IOstreams/IOstreams/IOstream.H | 24 ++++- src/OpenFOAM/primitives/Scalar/Scalar.C | 4 +- .../Scalar/doubleScalar/doubleScalar.H | 3 +- .../Scalar/floatScalar/floatScalar.H | 3 +- .../primitives/Scalar/scalar/scalar.C | 88 ++++++++++++++++- .../primitives/Scalar/scalar/scalar.H | 33 ++++++- src/OpenFOAM/primitives/ints/label/label.C | 95 +++++++++++++++++-- src/OpenFOAM/primitives/ints/label/label.H | 27 ++++-- 10 files changed, 356 insertions(+), 53 deletions(-) diff --git a/applications/test/limits/Test-limits.C b/applications/test/limits/Test-limits.C index ef6d1dc86c..8185e7c61f 100644 --- a/applications/test/limits/Test-limits.C +++ b/applications/test/limits/Test-limits.C @@ -30,6 +30,7 @@ Description #include "int.H" #include "uint.H" #include "string.H" +#include "scalar.H" #include "IOstreams.H" using namespace Foam; @@ -40,32 +41,54 @@ using namespace Foam; int main(int argc, char *argv[]) { //NONE Info<<"int16:" << pTraits::max << nl; - Info<<"=max=" << nl; - Info<<"uint8:" << std::numeric_limits::max() << nl; - Info<<"int16:" << std::numeric_limits::max() << nl; - Info<<"int32:" << pTraits::max << nl; - Info<<"uint32:" << pTraits::max << nl; - Info<<"int64:" << pTraits::max << nl; - Info<<"uint64:" << pTraits::max << nl; + Info<< "=max=" << nl; + Info<< "uint8:" << std::numeric_limits::max() << nl; + Info<< "int16:" << std::numeric_limits::max() << nl; + Info<< "int32:" << pTraits::max << nl; + Info<< "uint32:" << pTraits::max << nl; + Info<< "int64:" << pTraits::max << nl; + Info<< "uint64:" << pTraits::max << nl; Info<< nl; - cout<<"int16:" << std::numeric_limits::max() << nl; - cout<<"int32:" << pTraits::max << nl; - cout<<"uint32:" << pTraits::max << nl; - cout<<"int64:" << pTraits::max << nl; - cout<<"uint64:" << pTraits::max << nl; + cout<< "int16:" << std::numeric_limits::max() << nl; + cout<< "int32:" << pTraits::max << nl; + cout<< "uint32:" << pTraits::max << nl; + cout<< "int64:" << pTraits::max << nl; + cout<< "uint64:" << pTraits::max << nl; Info<< nl << "=digits=" << nl; - cout<<"int16:" << std::numeric_limits::digits << nl; - cout<<"int32:" << std::numeric_limits::digits << nl; - cout<<"uint32:" << std::numeric_limits::digits << nl; - cout<<"int64:" << std::numeric_limits::digits << nl; - cout<<"uint64:" << std::numeric_limits::digits << nl; - cout<<"float:" << std::numeric_limits::digits << nl; - cout<<"double:" << std::numeric_limits::digits << nl; - cout<<"long double:" << std::numeric_limits::digits << nl; + cout<< "int16:" << std::numeric_limits::digits << nl; + cout<< "int32:" << std::numeric_limits::digits << nl; + cout<< "uint32:" << std::numeric_limits::digits << nl; + cout<< "int64:" << std::numeric_limits::digits << nl; + cout<< "uint64:" << std::numeric_limits::digits << nl; + cout<< "float:" << std::numeric_limits::digits << nl; + cout<< "double:" << std::numeric_limits::digits << nl; + cout<< "long double:" << std::numeric_limits::digits << nl; + + Info<< nl << "=float=" << nl; + + cout<< "max:" << std::numeric_limits::max() + << " VGREAT:" << floatScalarVGREAT << nl; + cout<< "min:" << std::numeric_limits::min() + << " VSMALL:" << floatScalarVSMALL << nl; + cout<< "epsilon:" << std::numeric_limits::epsilon() + << " SMALL:" << floatScalarSMALL << nl; + cout<< "1/epsilon:" << 1.0f/std::numeric_limits::epsilon() + << " GREAT:" << floatScalarGREAT << nl; + + Info<< nl << "=double=" << nl; + + cout<< "max:" << std::numeric_limits::max() + << " VGREAT:" << doubleScalarVGREAT << nl; + cout<< "min:" << std::numeric_limits::min() + << " VSMALL:" << doubleScalarVSMALL << nl; + cout<< "epsilon:" << std::numeric_limits::epsilon() + << " SMALL:" << doubleScalarSMALL << nl; + cout<< "1/epsilon:" << 1.0f/std::numeric_limits::epsilon() + << " GREAT:" << doubleScalarGREAT << nl; Info << "---\nEnd\n" << endl; diff --git a/applications/test/primitives/Test-primitives.C b/applications/test/primitives/Test-primitives.C index 1b11d166eb..e46e700f20 100644 --- a/applications/test/primitives/Test-primitives.C +++ b/applications/test/primitives/Test-primitives.C @@ -30,7 +30,8 @@ Description \*---------------------------------------------------------------------------*/ #include "scalar.H" -#include "label.H" +#include "FlatOutput.H" +#include "ListStream.H" #include "StringStream.H" #include "NASCore.H" #include "parsing.H" @@ -67,6 +68,19 @@ void printInfo(const Switch& sw) } +Ostream& toString(Ostream& os, const UList& list) +{ + os << '"'; + for (const char c : list) + { + os << c; + } + os << '"'; + + return os; +} + + // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // template @@ -280,6 +294,59 @@ int main(int argc, char *argv[]) ); } + if (true) + { + #ifdef WM_DP + typedef float otherType; + #else + typedef double otherType; + #endif + + Info<< nl << "Test raw binary read of scalar list:" + << " write " << sizeof(otherType) + << " read " << sizeof(scalar) << nl; + + List srcList(15); + forAll(srcList, i) + { + srcList[i] = 1 + 10*i; + } + + DynamicList buf; + + OListStream os(std::move(buf), IOstream::BINARY); + os << srcList; + + os.swap(buf); // Recover buffer + + // Read back + List dstList; + + UIListStream is(buf, IOstream::BINARY); + is.setScalarByteSize(sizeof(otherType)); + + Info<< "Stream scalar-size (" + << is.scalarByteSize() << ") is native: " + << Switch(is.checkScalarSize()) << nl; + + + token firstToken(is); + Info<< "List has " << firstToken.info() << " scalar items" << nl; + + dstList.resize(firstToken.labelToken(), 3.14159); + + is.beginRawRead(); + // for (scalar& val : dstList) + // { + // val = readRawScalar(is); + // } + readRawScalars(is, dstList.data(), dstList.size()); + is.endRawRead(); + + Info<< "Wrote " << flatOutput(srcList) << nl + << "Read " << flatOutput(dstList) << nl; + } + if (nFail) { Info<< nl << "failed " << nFail << " tests" << nl; diff --git a/src/OpenFOAM/db/IOstreams/IOstreams/IOstream.H b/src/OpenFOAM/db/IOstreams/IOstreams/IOstream.H index 2b8e0fdeeb..642629bdf6 100644 --- a/src/OpenFOAM/db/IOstreams/IOstreams/IOstream.H +++ b/src/OpenFOAM/db/IOstreams/IOstreams/IOstream.H @@ -249,7 +249,7 @@ public: } - // Stream State Functions + // Element sizes (precision) //- The label byte-size associated with the stream unsigned labelByteSize() const @@ -275,6 +275,28 @@ public: scalarByteSize_ = nbytes; } + + //- Check if the label byte-size associated with the stream + //- is the same as the given type + template + typename std::enable_if::value, bool>::type + checkLabelSize() const + { + return labelByteSize_ == sizeof(T); + } + + //- Check if the scalar byte-size associated with the stream + //- is the same as the given type + template + typename std::enable_if::value, bool>::type + checkScalarSize() const + { + return scalarByteSize_ == sizeof(T); + } + + + // Stream State Functions + //- Const access to the current stream line number label lineNumber() const { diff --git a/src/OpenFOAM/primitives/Scalar/Scalar.C b/src/OpenFOAM/primitives/Scalar/Scalar.C index de61336a83..02da350ba3 100644 --- a/src/OpenFOAM/primitives/Scalar/Scalar.C +++ b/src/OpenFOAM/primitives/Scalar/Scalar.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2016-2018 OpenCFD Ltd. + \\ / A nd | Copyright (C) 2016-2019 OpenCFD Ltd. \\/ M anipulation | ------------------------------------------------------------------------------- | Copyright (C) 2011-2016 OpenFOAM Foundation @@ -123,7 +123,7 @@ bool ScalarRead(const char* buf, Scalar& val) Scalar ScalarRead(Istream& is) { - Scalar val; + Scalar val(0); is >> val; return val; diff --git a/src/OpenFOAM/primitives/Scalar/doubleScalar/doubleScalar.H b/src/OpenFOAM/primitives/Scalar/doubleScalar/doubleScalar.H index 4501dda96a..b3380a61e3 100644 --- a/src/OpenFOAM/primitives/Scalar/doubleScalar/doubleScalar.H +++ b/src/OpenFOAM/primitives/Scalar/doubleScalar/doubleScalar.H @@ -51,8 +51,7 @@ namespace Foam typedef double doubleScalar; // Largest and smallest scalar values allowed in certain parts of the code. -// (15 is the number of significant figures in an -// IEEE double precision number. See limits.h or float.h) +// See std::numeric_limits max(), min(), epsilon() constexpr doubleScalar doubleScalarGREAT = 1.0e+15; constexpr doubleScalar doubleScalarVGREAT = 1.0e+300; constexpr doubleScalar doubleScalarROOTVGREAT = 1.0e+150; diff --git a/src/OpenFOAM/primitives/Scalar/floatScalar/floatScalar.H b/src/OpenFOAM/primitives/Scalar/floatScalar/floatScalar.H index a8721b1d79..414d8d9744 100644 --- a/src/OpenFOAM/primitives/Scalar/floatScalar/floatScalar.H +++ b/src/OpenFOAM/primitives/Scalar/floatScalar/floatScalar.H @@ -51,8 +51,7 @@ namespace Foam typedef float floatScalar; // Largest and smallest scalar values allowed in certain parts of the code. -// (6 is the number of significant figures in an -// IEEE single precision number. See limits.h or float.h) +// See std::numeric_limits max(), min(), epsilon() constexpr floatScalar floatScalarGREAT = 1.0e+6; constexpr floatScalar floatScalarVGREAT = 1.0e+37; constexpr floatScalar floatScalarROOTVGREAT = 1.0e+18; diff --git a/src/OpenFOAM/primitives/Scalar/scalar/scalar.C b/src/OpenFOAM/primitives/Scalar/scalar/scalar.C index 99bb03a7b5..7a60f9911b 100644 --- a/src/OpenFOAM/primitives/Scalar/scalar/scalar.C +++ b/src/OpenFOAM/primitives/Scalar/scalar/scalar.C @@ -2,10 +2,8 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | + \\ / A nd | Copyright (C) 2004-2019 OpenCFD Ltd. \\/ M anipulation | -------------------------------------------------------------------------------- - | Copyright (C) 2011 OpenFOAM Foundation ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -32,11 +30,93 @@ License Foam::scalar Foam::readScalar(Istream& is) { - scalar val; + scalar val(0); is >> val; return val; } +Foam::scalar Foam::readRawScalar(Istream& is) +{ + scalar val(0); + readRawScalars(is, &val, 1); + return val; +} + + +void Foam::readRawScalars(Istream& is, scalar* data, size_t nElem) +{ + // No check for binary vs ascii, the caller knows what they are doing + + #if defined(WM_SP) || defined(WM_SPDP) + + // Defined scalar as a float, non-native type is double + // Handle type narrowing limits + + typedef double nonNative; + + if (is.checkScalarSize()) + { + nonNative other; + + for (const scalar* endData = data + nElem; data != endData; ++data) + { + is.readRaw(reinterpret_cast(&other), sizeof(nonNative)); + + // Type narrowing + // Overflow: silently fix, or raise error? + + if (other < -VGREAT) + { + *data = -VGREAT; + } + else if (other > VGREAT) + { + *data = VGREAT; + } + else if (other > -VSMALL && other < VSMALL) + { + // Underflow: round to zero + *data = 0; + } + else + { + *data = scalar(other); + } + } + } + else + { + // Read with native size + is.readRaw(reinterpret_cast(data), nElem*sizeof(scalar)); + } + + #elif defined(WM_DP) + + // Defined scalar as a double, non-native type is float + + typedef float nonNative; + + if (is.checkScalarSize()) + { + nonNative other; + + for (const scalar* endData = data + nElem; data != endData; ++data) + { + is.readRaw(reinterpret_cast(&other), sizeof(nonNative)); + + *data = scalar(other); + } + } + else + { + // Read with native size + is.readRaw(reinterpret_cast(data), nElem*sizeof(scalar)); + } + + #endif +} + + // ************************************************************************* // diff --git a/src/OpenFOAM/primitives/Scalar/scalar/scalar.H b/src/OpenFOAM/primitives/Scalar/scalar/scalar.H index cc9009df83..f0844babda 100644 --- a/src/OpenFOAM/primitives/Scalar/scalar/scalar.H +++ b/src/OpenFOAM/primitives/Scalar/scalar/scalar.H @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2017 OpenCFD Ltd. + \\ / A nd | Copyright (C) 2017-2019 OpenCFD Ltd. \\/ M anipulation | ------------------------------------------------------------------------------- | Copyright (C) 2011-2016 OpenFOAM Foundation @@ -64,8 +64,6 @@ namespace Foam constexpr scalar VSMALL = floatScalarVSMALL; constexpr scalar ROOTVSMALL = floatScalarROOTVSMALL; - scalar readScalar(Istream& is); - inline scalar readScalar(const char* buf) { return readFloat(buf); @@ -85,6 +83,19 @@ namespace Foam { return readFloat(str, val); } + + //- Read scalar from stream. + scalar readScalar(Istream& is); + + //- Read raw scalar from binary stream. + // \note No internal check for binary vs ascii, + // the caller knows what they are doing + scalar readRawScalar(Istream& is); + + //- Read raw scalars from binary stream. + // \note No internal check for binary vs ascii, + // the caller knows what they are doing + void readRawScalars(Istream& is, scalar* data, size_t nElem); } #elif defined(WM_DP) @@ -104,8 +115,6 @@ namespace Foam constexpr scalar VSMALL = doubleScalarVSMALL; constexpr scalar ROOTVSMALL = doubleScalarROOTVSMALL; - scalar readScalar(Istream& is); - inline scalar readScalar(const char* buf) { return readDouble(buf); @@ -125,6 +134,20 @@ namespace Foam { return readDouble(str, val); } + + + //- Read scalar from stream. + scalar readScalar(Istream& is); + + //- Read raw scalar from binary stream. + // \note No internal check for binary vs ascii, + // the caller knows what they are doing + scalar readRawScalar(Istream& is); + + //- Read raw scalars from binary stream. + // \note No internal check for binary vs ascii, + // the caller knows what they are doing + void readRawScalars(Istream& is, scalar* data, size_t nElem); } #endif diff --git a/src/OpenFOAM/primitives/ints/label/label.C b/src/OpenFOAM/primitives/ints/label/label.C index 07b9ccaa0f..e0c7afb2da 100644 --- a/src/OpenFOAM/primitives/ints/label/label.C +++ b/src/OpenFOAM/primitives/ints/label/label.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | + \\ / A nd | Copyright (C) 2019 OpenCFD Ltd. \\/ M anipulation | ------------------------------------------------------------------------------- | Copyright (C) 2011-2015 OpenFOAM Foundation @@ -27,16 +27,95 @@ License #include "error.H" #include "label.H" +#include "Istream.H" + +// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * // + +#if WM_LABEL_SIZE == 32 +const char* const Foam::pTraits::typeName = "label"; +const char* const Foam::pTraits::typeName = "int64"; +#elif WM_LABEL_SIZE == 64 +const char* const Foam::pTraits::typeName = "int32"; +const char* const Foam::pTraits::typeName = "label"; +#endif + // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // -#if WM_LABEL_SIZE == 32 -const char* const Foam::pTraits::typeName = "int64"; -const char* const Foam::pTraits::typeName = "label"; -#elif WM_LABEL_SIZE == 64 -const char* const Foam::pTraits::typeName = "label"; -const char* const Foam::pTraits::typeName = "int32"; -#endif +Foam::label Foam::readRawLabel(Istream& is) +{ + label val(0); + readRawLabels(is, &val, 1); + return val; +} + + +void Foam::readRawLabels(Istream& is, label* data, size_t nElem) +{ + // No check for binary vs ascii, the caller knows what they are doing + + #if WM_LABEL_SIZE == 32 + + // Defined label as int32, non-native type is int64 + // Handle type narrowing limits + + typedef int64_t nonNative; + + if (is.checkLabelSize()) + { + nonNative parsed; + + for (const label* endData = data + nElem; data != endData; ++data) + { + is.readRaw(reinterpret_cast(&parsed), sizeof(nonNative)); + + // Type narrowing + // Overflow: silently fix, or raise error? + if (parsed < labelMin) + { + *data = labelMin; + } + else if (parsed > labelMax) + { + *data = labelMax; + } + else + { + *data = label(parsed); + } + } + } + else + { + // Read with native size + is.readRaw(reinterpret_cast(data), nElem*sizeof(label)); + } + + #elif WM_LABEL_SIZE == 64 + + // Defined label as int64, non-native type is int32 + + typedef int32_t nonNative; + + if (is.checkLabelSize()) + { + nonNative parsed; + + for (const label* endData = data + nElem; data != endData; ++data) + { + is.readRaw(reinterpret_cast(&parsed), sizeof(nonNative)); + + *data = label(parsed); + } + } + else + { + // Read with native size + is.readRaw(reinterpret_cast(data), nElem*sizeof(label)); + } + + #endif +} // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // diff --git a/src/OpenFOAM/primitives/ints/label/label.H b/src/OpenFOAM/primitives/ints/label/label.H index 57359f3ffc..3e209a3dd3 100644 --- a/src/OpenFOAM/primitives/ints/label/label.H +++ b/src/OpenFOAM/primitives/ints/label/label.H @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2018 OpenCFD Ltd. + \\ / A nd | Copyright (C) 2018-2019 OpenCFD Ltd. \\/ M anipulation | ------------------------------------------------------------------------------- | Copyright (C) 2011-2014 OpenFOAM Foundation @@ -63,13 +63,6 @@ typedef INT_SIZE(int, _t) label; constexpr label labelMin = INT_SIZE(INT, _MIN); constexpr label labelMax = INT_SIZE(INT, _MAX); -//- Read label from stream. -// Uses readInt32 or readInt64 according to WM_LABEL_SIZE -inline label readLabel(Istream& is) -{ - return INT_SIZE(readInt,) (is); -} - //- Parse entire buffer as a label, skipping leading/trailing whitespace. // Uses readInt32 or readInt64 according to WM_LABEL_SIZE // \return Parsed value or FatalIOError on any problem @@ -103,6 +96,24 @@ inline bool readLabel(const std::string& str, label& val) } +//- Read label from stream. +// Uses readInt32 or readInt64 according to WM_LABEL_SIZE +inline label readLabel(Istream& is) +{ + return INT_SIZE(readInt,) (is); +} + +//- Read raw label from binary stream. +// \note No internal check for binary vs ascii, +// the caller knows what they are doing +label readRawLabel(Istream& is); + +//- Read raw labels from binary stream. +// \note No internal check for binary vs ascii, +// the caller knows what they are doing +void readRawLabels(Istream& is, label* data, size_t nElem); + + // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // //- Raise one label to the power of another