From c4de3e0a4d35cf006e6f10f656abbb988c2c7dd9 Mon Sep 17 00:00:00 2001 From: Mark Olesen Date: Sun, 5 Nov 2017 20:05:28 +0100 Subject: [PATCH] ENH: enhancements to behaviour of token - improved memory alignment reduces overhead for Int32 compilation - added move/swap semantics - made the type() readonly in favour of setVariant() to allow change of variant within a particular storage representation. Eg, STRING -> VERBATIMSTRING. --- applications/test/token/Make/files | 3 + applications/test/token/Make/options | 1 + applications/test/token/Test-token.C | 75 ++ applications/test/tokenize/Test-tokenize.C | 14 + src/OpenFOAM/db/IOstreams/IOstreams/Istream.C | 14 +- src/OpenFOAM/db/IOstreams/IOstreams/Istream.H | 8 +- .../db/IOstreams/Pstreams/UIPstream.C | 25 +- src/OpenFOAM/db/IOstreams/Sstreams/ISstream.C | 39 +- src/OpenFOAM/db/IOstreams/token/token.C | 22 +- src/OpenFOAM/db/IOstreams/token/token.H | 411 +++++++---- src/OpenFOAM/db/IOstreams/token/tokenI.H | 680 +++++++++++------- src/OpenFOAM/db/IOstreams/token/tokenIO.C | 285 ++++---- src/OpenFOAM/memory/refCount/refCount.H | 28 +- src/OpenFOAM/memory/tmp/tmp.H | 2 +- 14 files changed, 965 insertions(+), 642 deletions(-) create mode 100644 applications/test/token/Make/files create mode 100644 applications/test/token/Make/options create mode 100644 applications/test/token/Test-token.C diff --git a/applications/test/token/Make/files b/applications/test/token/Make/files new file mode 100644 index 0000000000..27b95da13e --- /dev/null +++ b/applications/test/token/Make/files @@ -0,0 +1,3 @@ +Test-token.C + +EXE = $(FOAM_USER_APPBIN)/Test-token diff --git a/applications/test/token/Make/options b/applications/test/token/Make/options new file mode 100644 index 0000000000..1f502ad153 --- /dev/null +++ b/applications/test/token/Make/options @@ -0,0 +1 @@ +/* EXE_INC = */ diff --git a/applications/test/token/Test-token.C b/applications/test/token/Test-token.C new file mode 100644 index 0000000000..b8918c54b7 --- /dev/null +++ b/applications/test/token/Test-token.C @@ -0,0 +1,75 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | OpenFOAM: The Open Source CFD Toolbox + \\ / O peration | + \\ / A nd | Copyright (C) 2017 OpenCFD Ltd. + \\/ M anipulation | +------------------------------------------------------------------------------- +License + This file is part of OpenFOAM. + + OpenFOAM is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with OpenFOAM. If not, see . + +Description + Test token construct assign etc. +\*---------------------------------------------------------------------------*/ + +#include "argList.H" +#include "IOobject.H" +#include "IOstreams.H" +#include "IFstream.H" +#include "StringStream.H" +#include "cpuTime.H" +#include "DynamicList.H" + +using namespace Foam; + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // +// Main program: + +int main(int argc, char *argv[]) +{ + argList::noBanner(); + argList::noParallel(); + + argList args(argc, argv, false, true); + + token tok1; + Info<< "construct null: " << tok1.info() << endl; + + tok1 = double(3.14159); + Info<< "assign double: " << tok1.info() << endl; + + token tok2(tok1); + Info<< "copy construct: " << tok2.info() << endl; + + tok1 = word("this-word"); + Info<< "assign word: " << tok1.info() << endl; + + token tok3(tok1); + Info<< "copy construct: " << tok3.info() << endl; + Info<< "orig: " << tok1.info() << endl; + + token tok4(std::move(tok1)); + Info<< "move construct: " << tok4.info() << endl; + Info<< "orig: " << tok1.info() << endl; + + tok3 = tok4; + Info<< "assign token: " << tok3.info() << endl; + Info<< "orig: " << tok4.info() << endl; + + return 0; +} + +// ************************************************************************* // diff --git a/applications/test/tokenize/Test-tokenize.C b/applications/test/tokenize/Test-tokenize.C index bd2f84a482..f944036d66 100644 --- a/applications/test/tokenize/Test-tokenize.C +++ b/applications/test/tokenize/Test-tokenize.C @@ -33,6 +33,7 @@ Description #include "IFstream.H" #include "StringStream.H" #include "cpuTime.H" +#include "DynamicList.H" using namespace Foam; @@ -69,6 +70,8 @@ int main(int argc, char *argv[]) IStringStream is(rawArg); + DynamicList tokens; + while (is.good()) { token tok(is); @@ -83,12 +86,23 @@ int main(int argc, char *argv[]) << " lookahead: '" << char(lookahead) << "'" << endl; } + + if (tok.good()) + { + tokens.append(std::move(tok)); + if (verbose) + { + Info<< "after append: " << tok.info() << endl; + } + } } if (verbose) { Info<< nl; IOobject::writeDivider(Info); + + Info<< "tokenList:" << tokens << endl; } } } diff --git a/src/OpenFOAM/db/IOstreams/IOstreams/Istream.C b/src/OpenFOAM/db/IOstreams/IOstreams/Istream.C index 25ee28e005..941b78cfd6 100644 --- a/src/OpenFOAM/db/IOstreams/IOstreams/Istream.C +++ b/src/OpenFOAM/db/IOstreams/IOstreams/Istream.C @@ -27,7 +27,7 @@ License // * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // -void Foam::Istream::putBack(const token& t) +void Foam::Istream::putBack(const token& tok) { if (bad()) { @@ -43,13 +43,13 @@ void Foam::Istream::putBack(const token& t) } else { - putBackToken_ = t; + putBackToken_ = tok; putBack_ = true; } } -bool Foam::Istream::getBack(token& t) +bool Foam::Istream::getBack(token& tok) { if (bad()) { @@ -59,7 +59,7 @@ bool Foam::Istream::getBack(token& t) } else if (putBack_) { - t = putBackToken_; + tok = putBackToken_; putBack_ = false; return true; } @@ -68,15 +68,15 @@ bool Foam::Istream::getBack(token& t) } -bool Foam::Istream::peekBack(token& t) +bool Foam::Istream::peekBack(token& tok) { if (putBack_) { - t = putBackToken_; + tok = putBackToken_; } else { - t = token::undefinedToken; + tok = token::undefinedToken; } return putBack_; diff --git a/src/OpenFOAM/db/IOstreams/IOstreams/Istream.H b/src/OpenFOAM/db/IOstreams/IOstreams/Istream.H index 24df823bdd..e55211309d 100644 --- a/src/OpenFOAM/db/IOstreams/IOstreams/Istream.H +++ b/src/OpenFOAM/db/IOstreams/IOstreams/Istream.H @@ -96,16 +96,16 @@ public: //- Put back token // Only a single put back is permitted - void putBack(const token&); + void putBack(const token& tok); //- Get the put back token if there is one and return true. // Return false if no put back token is available. - bool getBack(token&); + bool getBack(token& tok); //- Peek at the put back token without removing it. // Returns false if no put back token is available and set the // token to undefined. - bool peekBack(token&); + bool peekBack(token& tok); //- Return next token from stream virtual Istream& read(token&) = 0; @@ -116,7 +116,7 @@ public: //- Read a word virtual Istream& read(word&) = 0; - // Read a string (including enclosing double-quotes) + //- Read a string (including enclosing double-quotes) virtual Istream& read(string&) = 0; //- Read a label diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UIPstream.C b/src/OpenFOAM/db/IOstreams/Pstreams/UIPstream.C index 0e40098fa5..1ed965b758 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/UIPstream.C +++ b/src/OpenFOAM/db/IOstreams/Pstreams/UIPstream.C @@ -3,7 +3,7 @@ \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | \\ / A nd | Copyright (C) 2011-2015 OpenFOAM Foundation - \\/ M anipulation | + \\/ M anipulation | Copyright (C) 2017 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -164,22 +164,20 @@ Foam::Istream& Foam::UIPstream::read(token& t) // Word case token::tokenType::WORD : { - word* pval = new word; - if (read(*pval)) + word val; + if (read(val)) { - if (token::compound::isCompound(*pval)) + if (token::compound::isCompound(val)) { - t = token::compound::New(*pval, *this).ptr(); - delete pval; + t = token::compound::New(val, *this).ptr(); } else { - t = pval; + t = std::move(val); } } else { - delete pval; t.setBad(); } return *this; @@ -190,26 +188,25 @@ Foam::Istream& Foam::UIPstream::read(token& t) { // Recurse to read actual string read(t); - t.type() = token::tokenType::VERBATIMSTRING; + t.setType(token::tokenType::VERBATIMSTRING); return *this; } case token::tokenType::VARIABLE : { // Recurse to read actual string read(t); - t.type() = token::tokenType::VARIABLE; + t.setType(token::tokenType::VARIABLE); return *this; } case token::tokenType::STRING : { - string* pval = new string; - if (read(*pval)) + string val; + if (read(val)) { - t = pval; + t = std::move(val); } else { - delete pval; t.setBad(); } return *this; diff --git a/src/OpenFOAM/db/IOstreams/Sstreams/ISstream.C b/src/OpenFOAM/db/IOstreams/Sstreams/ISstream.C index 010eef975b..cc9a0e5a65 100644 --- a/src/OpenFOAM/db/IOstreams/Sstreams/ISstream.C +++ b/src/OpenFOAM/db/IOstreams/Sstreams/ISstream.C @@ -116,21 +116,18 @@ char Foam::ISstream::nextValid() void Foam::ISstream::readWordToken(token& t) { - word* wPtr = new word; - - if (read(*wPtr).bad()) + word val; + if (read(val).bad()) { - delete wPtr; t.setBad(); } - else if (token::compound::isCompound(*wPtr)) + else if (token::compound::isCompound(val)) { - t = token::compound::New(*wPtr, *this).ptr(); - delete wPtr; + t = token::compound::New(val, *this).ptr(); } else { - t = wPtr; // Token takes ownership + t = std::move(val); // Move contents to token } } @@ -192,16 +189,15 @@ Foam::Istream& Foam::ISstream::read(token& t) case token::BEGIN_STRING : { putback(c); - string* sPtr = new string; - if (read(*sPtr).bad()) + string val; + if (read(val).bad()) { - delete sPtr; t.setBad(); } else { - t = sPtr; // Token takes ownership + t = std::move(val); // Move contents to token } return *this; @@ -219,17 +215,16 @@ Foam::Istream& Foam::ISstream::read(token& t) else if (nextC == token::BEGIN_BLOCK) { // Verbatim string: #{ ... #} - string* sPtr = new string; - if (readVerbatim(*sPtr).bad()) + string val; + if (readVerbatim(val).bad()) { - delete sPtr; t.setBad(); } else { - t = sPtr; // Token takes ownership - t.type() = token::tokenType::VERBATIMSTRING; + t = std::move(val); // Move contents to token + t.setType(token::tokenType::VERBATIMSTRING); } } else @@ -259,17 +254,15 @@ Foam::Istream& Foam::ISstream::read(token& t) putback(nextC); putback(c); - string* sPtr = new string; - - if (readVariable(*sPtr).bad()) + string val; + if (readVariable(val).bad()) { - delete sPtr; t.setBad(); } else { - t = sPtr; // Token takes ownership - t.type() = token::tokenType::VARIABLE; + t = std::move(val); // Move contents to token + t.setType(token::tokenType::VARIABLE); } } else diff --git a/src/OpenFOAM/db/IOstreams/token/token.C b/src/OpenFOAM/db/IOstreams/token/token.C index 17019bda99..fe1f76df5c 100644 --- a/src/OpenFOAM/db/IOstreams/token/token.C +++ b/src/OpenFOAM/db/IOstreams/token/token.C @@ -3,7 +3,7 @@ \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | \\ / A nd | Copyright (C) 2011-2016 OpenFOAM Foundation - \\/ M anipulation | + \\/ M anipulation | Copyright (C) 2017 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -29,14 +29,14 @@ License namespace Foam { - const char* const token::typeName = "token"; - token token::undefinedToken; - typedef token::compound tokenCompound; defineTypeNameAndDebug(tokenCompound, 0); defineRunTimeSelectionTable(tokenCompound, Istream); } +const char* const Foam::token::typeName = "token"; +const Foam::token Foam::token::undefinedToken; + // * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * * // @@ -94,7 +94,7 @@ Foam::token::compound& Foam::token::transferCompoundToken(const Istream& is) { if (type_ == tokenType::COMPOUND) { - if (compoundTokenPtr_->empty()) + if (data_.compoundPtr->empty()) { FatalIOErrorInFunction(is) << "compound has already been transfered from token\n " @@ -102,16 +102,14 @@ Foam::token::compound& Foam::token::transferCompoundToken(const Istream& is) } else { - compoundTokenPtr_->empty() = true; + data_.compoundPtr->empty() = true; } - return *compoundTokenPtr_; - } - else - { - parseError("compound"); - return *compoundTokenPtr_; + return *data_.compoundPtr; } + + parseError("compound"); + return *data_.compoundPtr; } diff --git a/src/OpenFOAM/db/IOstreams/token/token.H b/src/OpenFOAM/db/IOstreams/token/token.H index 1de16e5e13..90897917c3 100644 --- a/src/OpenFOAM/db/IOstreams/token/token.H +++ b/src/OpenFOAM/db/IOstreams/token/token.H @@ -3,7 +3,7 @@ \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | \\ / A nd | Copyright (C) 2011-2016 OpenFOAM Foundation - \\/ M anipulation | + \\/ M anipulation | Copyright (C) 2017 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -25,7 +25,7 @@ Class Foam::token Description - A token holds items read from Istream. + A token holds an item read from Istream. SourceFiles tokenI.H @@ -64,34 +64,40 @@ Ostream& operator<<(Ostream& os, const token& t); /*---------------------------------------------------------------------------*\ - Class token Declaration + Class token Declaration \*---------------------------------------------------------------------------*/ class token { - public: - //- Enumeration defining the types of token + //- Enumeration defining the types of token. + // Since these values are also used to tag content in Pstream, + // the maximum number of types is limited to 30. enum tokenType { - UNDEFINED, + UNDEFINED, //!< An undefined token-type - PUNCTUATION, - WORD, - VARIABLE, - STRING, - VERBATIMSTRING, - LABEL, - FLOAT_SCALAR, - DOUBLE_SCALAR, - COMPOUND, + // Fundamental types + PUNCTUATION, //!< single character punctuation + LABEL, //!< label (integer) type + FLOAT_SCALAR, //!< float (single-precision) type + DOUBLE_SCALAR, //!< double (double-precision) type - ERROR + // Pointer types + WORD, //!< Contents represent a Foam::word + STRING, //!< Contents represent a Foam::string + VARIABLE, //!< Contents are a Foam::string representing a + //!< dictionary \c $variable expansion + VERBATIMSTRING, //!< Contents are a Foam::string representing verbatim + //!< content + COMPOUND, //!< Compound type such as List\ etc. + + ERROR //!< A token error encountered }; - //- Standard punctuation tokens + //- Standard punctuation tokens (a character) enum punctuationToken { NULL_TOKEN = '\0', @@ -99,25 +105,26 @@ public: TAB = '\t', NL = '\n', - END_STATEMENT = ';', - BEGIN_LIST = '(', - END_LIST = ')', - BEGIN_SQR = '[', - END_SQR = ']', - BEGIN_BLOCK = '{', - END_BLOCK = '}', - COLON = ':', - COMMA = ',', + END_STATEMENT = ';', //!< End entry [#isseparator] + BEGIN_LIST = '(', //!< Begin list [#isseparator] + END_LIST = ')', //!< End list [#isseparator] + BEGIN_SQR = '[', //!< Begin dimensions [#isseparator] + END_SQR = ']', //!< End dimensions [#isseparator] + BEGIN_BLOCK = '{', //!< Begin block [#isseparator] + END_BLOCK = '}', //!< End block [#isseparator] + COLON = ':', //!< Colon [#isseparator] + COMMA = ',', //!< Comma [#isseparator] HASH = '#', + ATSYM = '@', BEGIN_STRING = '"', END_STRING = BEGIN_STRING, - ASSIGN = '=', - ADD = '+', - SUBTRACT = '-', - MULTIPLY = '*', - DIVIDE = '/' + ASSIGN = '=', //!< Assigment/equals [#isseparator] + ADD = '+', //!< Addition [#isseparator] + SUBTRACT = '-', //!< Substract or start of negative number + MULTIPLY = '*', //!< Multiply [#isseparator] + DIVIDE = '/' //!< Divide [#isseparator] }; @@ -145,7 +152,6 @@ public: //- Runtime type information TypeName("compound"); - //- Declare run-time constructor selection table declareRunTimeSelectionTable ( @@ -169,7 +175,7 @@ public: // Selectors //- Select null constructed - static autoPtr New(const word& type, Istream&); + static autoPtr New(const word& type, Istream& is); //- Destructor @@ -178,26 +184,22 @@ public: // Member Functions - // Access + //- Return true if name is a known compound type + static bool isCompound(const word& name); - //- Return true if name is a compound type - static bool isCompound(const word& name); + bool empty() const + { + return empty_; + } - bool empty() const - { - return empty_; - } + bool& empty() + { + return empty_; + } - bool& empty() - { - return empty_; - } + virtual label size() const = 0; - virtual label size() const = 0; - - // Write - - virtual void write(Ostream& os) const = 0; + virtual void write(Ostream& os) const = 0; // IOstream Operators @@ -236,35 +238,49 @@ public: //- Static undefined token - static token undefinedToken; + static const token undefinedToken; private: + //- A %union of token types + union content + { + // Fundamental values. Largest first for any {} initialization. + int64_t int64Val; + int32_t int32Val; + + punctuationToken punctuationVal; + label labelVal; + floatScalar floatVal; + doubleScalar doubleVal; + + // Pointers + word* wordPtr; + string* stringPtr; + mutable compound* compoundPtr; + }; + + // Private data + //- The data content (as a union). + // For memory alignment this should appear as the first member. + content data_; + //- The token type tokenType type_; - //- Anonymous Union of token types - union - { - punctuationToken punctuationToken_; - word* wordTokenPtr_; - string* stringTokenPtr_; - label labelToken_; - floatScalar floatScalarToken_; - doubleScalar doubleScalarToken_; - mutable compound* compoundTokenPtr_; - }; - - //- Line number in the file this token was read from + //- Line number in the file the token was read from label lineNumber_; // Private Member Functions - //- Clear any allocated storage (word or string) + //- Set as UNDEFINED and zero the union content without any checking + inline void setUndefined(); + + //- Clear any allocated storage (word or string) and set to UNDEFINED inline void clear(); // Parse error, expected 'expected', found ... @@ -283,18 +299,15 @@ public: //- Construct null inline token(); - //- Construct as copy + //- Copy construct inline token(const token& t); + //- Move construct. The original token is left as UNDEFINED. + inline token(token&& t); + //- Construct punctuation character token inline explicit token(punctuationToken p); - //- Construct word token - inline explicit token(const word& w); - - //- Construct string token - inline explicit token(const string& str); - //- Construct label token inline explicit token(const label val); @@ -304,16 +317,16 @@ public: //- Construct doubleScalar token inline explicit token(const doubleScalar val); + //- Construct word token by copying word contents + inline explicit token(const word& w); + + //- Construct string token by copying string contents + inline explicit token(const string& str); + //- Construct punctuation character token inline token(punctuationToken p, const label lineNumber); - //- Construct word token - inline token(const word& w, const label lineNumber); - - //- Construct string token - inline token(const string& str, const label lineNumber); - //- Construct label token inline token(const label val, const label lineNumber); @@ -323,6 +336,12 @@ public: //- Construct doubleScalar token inline token(const doubleScalar val, const label lineNumber); + //- Construct word token by copying word contents + inline token(const word& w, const label lineNumber); + + //- Construct string token by copying string contents + inline token(const string& str, const label lineNumber); + //- Construct from Istream token(Istream& is); @@ -331,108 +350,216 @@ public: inline ~token(); + // Static member functions + + //- True if the character is a punctuation separator (eg, in ISstream). + // Since it could also start a number, SUBTRACT is not included as + // a separator. + // + // \param c the character to test, passed as int for consistency with + // isdigit, isspace etc. + inline static bool isseparator(int c); + + // Member functions - // Access + // Status - inline tokenType type() const; - inline tokenType& type(); + //- Return the name of the token type + word name() const; - inline bool good() const; - inline bool undefined() const; - inline bool error() const; + //- Return the token type + inline tokenType type() const; - inline bool isPunctuation() const; - inline punctuationToken pToken() const; + //- Change the token type, for similar types. + // This can be used to change between string-like variants + // (eg, STRING, VARIABLE, VERBATIMSTRING) + // To change types entirely (eg, STRING to DOUBLE_SCALAR), + // use the corresponding assignment operator. + // + // \return true if the change was successful or no change was required + inline bool setType(const tokenType variant); - inline bool isWord() const; - inline const word& wordToken() const; + //- The line number for the token + inline label lineNumber() const; - inline bool isVariable() const; + //- The line number for the token + inline label& lineNumber(); - inline bool isString() const; - inline const string& stringToken() const; + //- True if token is not UNDEFINED or ERROR + inline bool good() const; - inline bool isLabel() const; - inline label labelToken() const; + //- True if token is UNDEFINED + inline bool undefined() const; - inline bool isFloatScalar() const; - inline floatScalar floatScalarToken() const; + //- True if token is ERROR + inline bool error() const; - inline bool isDoubleScalar() const; - inline doubleScalar doubleScalarToken() const; + //- True if token is PUNCTUATION + inline bool isPunctuation() const; - inline bool isScalar() const; - inline scalar scalarToken() const; + //- True if token is PUNCTUATION and isseparator + inline bool isSeparator() const; - inline bool isNumber() const; - inline scalar number() const; + //- True if token is LABEL + inline bool isLabel() const; - inline bool isCompound() const; - inline const compound& compoundToken() const; - compound& transferCompoundToken(const Istream& is); + //- True if token is FLOAT_SCALAR + inline bool isFloatScalar() const; - inline label lineNumber() const; - inline label& lineNumber(); + //- True if token is DOUBLE_SCALAR + inline bool isDoubleScalar() const; + + //- True if token is FLOAT_SCALAR or DOUBLE_SCALAR + inline bool isScalar() const; + + //- True if token is LABEL, FLOAT_SCALAR or DOUBLE_SCALAR + inline bool isNumber() const; + + //- True if token is WORD + inline bool isWord() const; + + //- True if token is STRING, VARIABLE or VERBATIMSTRING + inline bool isString() const; + + //- True if token is VARIABLE + inline bool isVariable() const; + + //- True if token is COMPOUND + inline bool isCompound() const; - // Edit + // Access - //- Set bad - inline void setBad(); + //- Return punctuation character. + // Report FatalIOError and return \b \\0 if token is not PUNCTUATION + inline punctuationToken pToken() const; + + //- Return label value. + // Report FatalIOError and return \b 0 if token is not LABEL + inline label labelToken() const; + + //- Return float value. + // Report FatalIOError and return \b 0.0 if token is not FLOAT_SCALAR + inline floatScalar floatScalarToken() const; + + //- Return double value. + // Report FatalIOError and return \b 0.0 if token is not DOUBLE_SCALAR + inline doubleScalar doubleScalarToken() const; + + //- Return float or double value. + // Report FatalIOError and return \b 0.0 if token is not a + // FLOAT_SCALAR or DOUBLE_SCALAR + inline scalar scalarToken() const; + + //- Return label, float or double value. + // Report FatalIOError and return \b 0.0 if token is not a + // LABEL, FLOAT_SCALAR or DOUBLE_SCALAR + inline scalar number() const; + + //- Return const reference to the word contents. + // Report FatalIOError and return \b "" if token is not a WORD + inline const word& wordToken() const; + + //- Return const reference to the string contents. + // Report FatalIOError and return \b "" if token is not a + // STRING, VARIABLE or VERBATIMSTRING + inline const string& stringToken() const; + + //- Read access for compound token + inline const compound& compoundToken() const; + + //- Return reference to compound token and decrease its internal + //- refCound accordingly. + // The Istream is used for reference error messages only. + compound& transferCompoundToken(const Istream& is); - // Info + // Edit - //- Return info proxy. - // Used to print token information to a stream - InfoProxy info() const - { - return *this; - } + //- Clear token and set to be in an error state. + inline void setBad(); + + //- Swap token contents: type, data, line-number + inline void swap(token& tok); + + + // Info + + //- Return info proxy for printing token information to a stream + InfoProxy info() const + { + return *this; + } // Member operators - // Assignment + // Assignment - inline void operator=(const token& t); + //- Copy assign + inline void operator=(const token& tok); - inline void operator=(const punctuationToken p); + //- Move assign + inline void operator=(token&& tok); - inline void operator=(word* wPtr); - inline void operator=(const word& w); + //- Copy assign from punctuation + inline void operator=(const punctuationToken p); - inline void operator=(string* strPtr); - inline void operator=(const string& str); + //- Copy assign from label + inline void operator=(const label val); - inline void operator=(const label val); - inline void operator=(const floatScalar val); - inline void operator=(const doubleScalar val); + //- Copy assign from float + inline void operator=(const floatScalar val); - inline void operator=(compound* compPtr); + //- Copy assign from double + inline void operator=(const doubleScalar val); + + //- Copy assign from word + inline void operator=(const word& w); + + //- Copy assign from string + inline void operator=(const string& str); + + //- Move assign from word + inline void operator=(word&& w); + + //- Move assign from string + inline void operator=(string&& str); + + //- Transfer word pointer to the token + // \deprecated in favour of using move assign from word + inline void operator=(word* wordPtr); + + //- Transfer string pointer to the token + // \deprecated in favour of using move assign from string + inline void operator=(string* stringPtr); + + //- Assign compound with reference counting to token + inline void operator=(compound* compoundPtr); - // Equality + // Equality - inline bool operator==(const token& t) const; - inline bool operator==(const punctuationToken p) const; - inline bool operator==(const word& w) const; - inline bool operator==(const string& str) const; - inline bool operator==(const label val) const; - inline bool operator==(const floatScalar val) const; - inline bool operator==(const doubleScalar val) const; + inline bool operator==(const token& t) const; + + inline bool operator==(const punctuationToken p) const; + inline bool operator==(const label val) const; + inline bool operator==(const floatScalar val) const; + inline bool operator==(const doubleScalar val) const; + inline bool operator==(const word& w) const; + inline bool operator==(const string& str) const; - // Inequality + // Inequality - inline bool operator!=(const token& t) const; - inline bool operator!=(const punctuationToken p) const; - inline bool operator!=(const word& w) const; - inline bool operator!=(const string& str) const; - inline bool operator!=(const label val) const; - inline bool operator!=(const floatScalar val) const; - inline bool operator!=(const doubleScalar val) const; + inline bool operator!=(const token& t) const; + inline bool operator!=(const punctuationToken p) const; + inline bool operator!=(const label val) const; + inline bool operator!=(const floatScalar val) const; + inline bool operator!=(const doubleScalar val) const; + inline bool operator!=(const word& w) const; + inline bool operator!=(const string& str) const; // IOstream operators diff --git a/src/OpenFOAM/db/IOstreams/token/tokenI.H b/src/OpenFOAM/db/IOstreams/token/tokenI.H index 32417b4e75..c391298652 100644 --- a/src/OpenFOAM/db/IOstreams/token/tokenI.H +++ b/src/OpenFOAM/db/IOstreams/token/tokenI.H @@ -3,7 +3,7 @@ \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | \\ / A nd | Copyright (C) 2011-2016 OpenFOAM Foundation - \\/ M anipulation | + \\/ M anipulation | Copyright (C) 2017 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -23,36 +23,54 @@ License \*---------------------------------------------------------------------------*/ +#include + // * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * // +inline void Foam::token::setUndefined() +{ + type_ = tokenType::UNDEFINED; + data_.int64Val = 0; // bit-wise zero for union content + // leave lineNumber untouched - may still be needed +} + + inline void Foam::token::clear() { - if (type_ == tokenType::WORD) + switch (type_) { - delete wordTokenPtr_; - } - else if - ( - type_ == tokenType::STRING - || type_ == tokenType::VARIABLE - || type_ == tokenType::VERBATIMSTRING - ) - { - delete stringTokenPtr_; - } - else if (type_ == tokenType::COMPOUND) - { - if (compoundTokenPtr_->unique()) + case tokenType::WORD: { - delete compoundTokenPtr_; + delete data_.wordPtr; + break; } - else + + case tokenType::STRING: + case tokenType::VARIABLE: + case tokenType::VERBATIMSTRING: { - compoundTokenPtr_->refCount::operator--(); + delete data_.stringPtr; + break; } + + case tokenType::COMPOUND: + { + if (data_.compoundPtr->unique()) + { + delete data_.compoundPtr; + } + else + { + data_.compoundPtr->refCount::operator--(); + } + break; + } + + default: + break; } - type_ = tokenType::UNDEFINED; + setUndefined(); } @@ -60,58 +78,62 @@ inline void Foam::token::clear() inline Foam::token::token() : + data_(), // bit-wise zero for union content type_(tokenType::UNDEFINED), lineNumber_(0) {} -inline Foam::token::token(const token& t) +inline Foam::token::token(const token& tok) : - type_(t.type_), - lineNumber_(t.lineNumber_) + data_(tok.data_), // bit-wise copy of union content + type_(tok.type_), + lineNumber_(tok.lineNumber_) { + // Fundamental: values already handled by bit-wise copy + // Pointer: duplicate content or increase refCount + switch (type_) { - case tokenType::UNDEFINED: - break; - - case tokenType::PUNCTUATION: - punctuationToken_ = t.punctuationToken_; - break; - case tokenType::WORD: - wordTokenPtr_ = new word(*t.wordTokenPtr_); - break; + { + data_.wordPtr = new word(*tok.data_.wordPtr); + break; + } case tokenType::STRING: case tokenType::VARIABLE: case tokenType::VERBATIMSTRING: - stringTokenPtr_ = new string(*t.stringTokenPtr_); - break; - - case tokenType::LABEL: - labelToken_ = t.labelToken_; - break; - - case tokenType::FLOAT_SCALAR: - floatScalarToken_ = t.floatScalarToken_; - break; - - case tokenType::DOUBLE_SCALAR: - doubleScalarToken_ = t.doubleScalarToken_; - break; + { + data_.stringPtr = new string(*tok.data_.stringPtr); + break; + } case tokenType::COMPOUND: - compoundTokenPtr_ = t.compoundTokenPtr_; - compoundTokenPtr_->refCount::operator++(); - break; + { + // Identical pointers, but increase the refCount + data_.compoundPtr = tok.data_.compoundPtr; + data_.compoundPtr->refCount::operator++(); + break; + } - case tokenType::ERROR: - break; + default: + break; } } +inline Foam::token::token(token&& tok) +: + data_(tok.data_), // bit-wise copy of union content + type_(tok.type_), + lineNumber_(tok.lineNumber_) +{ + tok.setUndefined(); // zero the union content without any checking + tok.lineNumber_ = 0; +} + + inline Foam::token::token(punctuationToken p) : token(p, 0) @@ -150,50 +172,62 @@ inline Foam::token::token(const doubleScalar val) inline Foam::token::token(punctuationToken p, label lineNumber) : + data_(), type_(tokenType::PUNCTUATION), - punctuationToken_(p), lineNumber_(lineNumber) -{} - - -inline Foam::token::token(const word& w, label lineNumber) -: - type_(tokenType::WORD), - wordTokenPtr_(new word(w)), - lineNumber_(lineNumber) -{} - - -inline Foam::token::token(const string& str, label lineNumber) -: - type_(tokenType::STRING), - stringTokenPtr_(new string(str)), - lineNumber_(lineNumber) -{} +{ + data_.punctuationVal = p; +} inline Foam::token::token(const label val, label lineNumber) : + data_(), type_(tokenType::LABEL), - labelToken_(val), lineNumber_(lineNumber) -{} +{ + data_.labelVal = val; +} inline Foam::token::token(const floatScalar val, label lineNumber) : + data_(), type_(tokenType::FLOAT_SCALAR), - floatScalarToken_(val), lineNumber_(lineNumber) -{} +{ + data_.floatVal = val; +} inline Foam::token::token(const doubleScalar val, label lineNumber) : + data_(), type_(tokenType::DOUBLE_SCALAR), - doubleScalarToken_(val), lineNumber_(lineNumber) -{} +{ + data_.doubleVal = val; +} + + +inline Foam::token::token(const word& w, label lineNumber) +: + data_(), + type_(tokenType::WORD), + lineNumber_(lineNumber) +{ + data_.wordPtr = new word(w); +} + + +inline Foam::token::token(const string& str, label lineNumber) +: + data_(), + type_(tokenType::STRING), + lineNumber_(lineNumber) +{ + data_.stringPtr = new string(str); +} // * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * // @@ -206,134 +240,179 @@ inline Foam::token::~token() // * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // -inline Foam::token::tokenType Foam::token::type() const +inline void Foam::token::swap(token& tok) +{ + std::swap(data_, tok.data_); + std::swap(type_, tok.type_); + std::swap(lineNumber_, tok.lineNumber_); +} + + +inline Foam::token::tokenType Foam::token::type() const { return type_; } -inline Foam::token::tokenType& Foam::token::type() + +inline bool Foam::token::setType(token::tokenType variant) { - return type_; + if (type_ == variant) + { + // No change required + return true; + } + + switch (variant) + { + case tokenType::STRING: + case tokenType::VARIABLE: + case tokenType::VERBATIMSTRING: + { + switch (type_) + { + // could also go from WORD to STRING etc - to be decided + case tokenType::STRING: + case tokenType::VARIABLE: + case tokenType::VERBATIMSTRING: + type_ = variant; + return true; + break; + + default: + return false; + break; + } + } + + default: + break; + } + + return false; } + +inline Foam::label Foam::token::lineNumber() const +{ + return lineNumber_; +} + + +inline Foam::label& Foam::token::lineNumber() +{ + return lineNumber_; +} + + inline bool Foam::token::good() const { - return (type_ != tokenType::ERROR && type_ != tokenType::UNDEFINED); + return (type_ != tokenType::UNDEFINED && type_ != tokenType::ERROR); } + inline bool Foam::token::undefined() const { return (type_ == tokenType::UNDEFINED); } + inline bool Foam::token::error() const { return (type_ == tokenType::ERROR); } + inline bool Foam::token::isPunctuation() const { return (type_ == tokenType::PUNCTUATION); } -inline Foam::token::punctuationToken Foam::token::pToken() const + +inline Foam::token::punctuationToken Foam::token::pToken() const { if (type_ == tokenType::PUNCTUATION) { - return punctuationToken_; - } - else - { - parseError("punctuation character"); - return NULL_TOKEN; + return data_.punctuationVal; } + + parseError("punctuation character"); + return NULL_TOKEN; } -inline bool Foam::token::isWord() const + +inline bool Foam::token::isseparator(int c) { - return (type_ == tokenType::WORD); -} - -inline const Foam::word& Foam::token::wordToken() const -{ - if (type_ == tokenType::WORD) + switch (c) { - return *wordTokenPtr_; - } - else - { - parseError(word::typeName); - return word::null; + case token::END_STATEMENT : + case token::BEGIN_LIST : + case token::END_LIST : + case token::BEGIN_SQR : + case token::END_SQR : + case token::BEGIN_BLOCK : + case token::END_BLOCK : + case token::COLON : + case token::COMMA : + case token::ASSIGN : + case token::ADD : + // Excluded token::SUBTRACT since it could start a number + case token::MULTIPLY : + case token::DIVIDE : + { + return true; + } + + default: + break; } + + return false; } -inline bool Foam::token::isVariable() const -{ - return (type_ == tokenType::VARIABLE); -} -inline bool Foam::token::isString() const +inline bool Foam::token::isSeparator() const { return ( - type_ == tokenType::STRING - || type_ == tokenType::VARIABLE - || type_ == tokenType::VERBATIMSTRING + type_ == tokenType::PUNCTUATION + && isseparator(data_.punctuationVal) ); } -inline const Foam::string& Foam::token::stringToken() const -{ - if - ( - type_ == tokenType::STRING - || type_ == tokenType::VARIABLE - || type_ == tokenType::VERBATIMSTRING - ) - { - return *stringTokenPtr_; - } - else - { - parseError(string::typeName); - return string::null; - } -} inline bool Foam::token::isLabel() const { return (type_ == tokenType::LABEL); } + inline Foam::label Foam::token::labelToken() const { if (type_ == tokenType::LABEL) { - return labelToken_; - } - else - { - parseError(pTraits