diff --git a/applications/test/token/Make/files b/applications/test/token/Make/files
new file mode 100644
index 0000000000..27b95da13e
--- /dev/null
+++ b/applications/test/token/Make/files
@@ -0,0 +1,3 @@
+Test-token.C
+
+EXE = $(FOAM_USER_APPBIN)/Test-token
diff --git a/applications/test/token/Make/options b/applications/test/token/Make/options
new file mode 100644
index 0000000000..1f502ad153
--- /dev/null
+++ b/applications/test/token/Make/options
@@ -0,0 +1 @@
+/* EXE_INC = */
diff --git a/applications/test/token/Test-token.C b/applications/test/token/Test-token.C
new file mode 100644
index 0000000000..b8918c54b7
--- /dev/null
+++ b/applications/test/token/Test-token.C
@@ -0,0 +1,75 @@
+/*---------------------------------------------------------------------------*\
+ ========= |
+ \\ / F ield | OpenFOAM: The Open Source CFD Toolbox
+ \\ / O peration |
+ \\ / A nd | Copyright (C) 2017 OpenCFD Ltd.
+ \\/ M anipulation |
+-------------------------------------------------------------------------------
+License
+ This file is part of OpenFOAM.
+
+ OpenFOAM is free software: you can redistribute it and/or modify it
+ under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with OpenFOAM. If not, see .
+
+Description
+ Test token construct assign etc.
+\*---------------------------------------------------------------------------*/
+
+#include "argList.H"
+#include "IOobject.H"
+#include "IOstreams.H"
+#include "IFstream.H"
+#include "StringStream.H"
+#include "cpuTime.H"
+#include "DynamicList.H"
+
+using namespace Foam;
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+// Main program:
+
+int main(int argc, char *argv[])
+{
+ argList::noBanner();
+ argList::noParallel();
+
+ argList args(argc, argv, false, true);
+
+ token tok1;
+ Info<< "construct null: " << tok1.info() << endl;
+
+ tok1 = double(3.14159);
+ Info<< "assign double: " << tok1.info() << endl;
+
+ token tok2(tok1);
+ Info<< "copy construct: " << tok2.info() << endl;
+
+ tok1 = word("this-word");
+ Info<< "assign word: " << tok1.info() << endl;
+
+ token tok3(tok1);
+ Info<< "copy construct: " << tok3.info() << endl;
+ Info<< "orig: " << tok1.info() << endl;
+
+ token tok4(std::move(tok1));
+ Info<< "move construct: " << tok4.info() << endl;
+ Info<< "orig: " << tok1.info() << endl;
+
+ tok3 = tok4;
+ Info<< "assign token: " << tok3.info() << endl;
+ Info<< "orig: " << tok4.info() << endl;
+
+ return 0;
+}
+
+// ************************************************************************* //
diff --git a/applications/test/tokenize/Test-tokenize.C b/applications/test/tokenize/Test-tokenize.C
index bd2f84a482..f944036d66 100644
--- a/applications/test/tokenize/Test-tokenize.C
+++ b/applications/test/tokenize/Test-tokenize.C
@@ -33,6 +33,7 @@ Description
#include "IFstream.H"
#include "StringStream.H"
#include "cpuTime.H"
+#include "DynamicList.H"
using namespace Foam;
@@ -69,6 +70,8 @@ int main(int argc, char *argv[])
IStringStream is(rawArg);
+ DynamicList tokens;
+
while (is.good())
{
token tok(is);
@@ -83,12 +86,23 @@ int main(int argc, char *argv[])
<< " lookahead: '" << char(lookahead) << "'"
<< endl;
}
+
+ if (tok.good())
+ {
+ tokens.append(std::move(tok));
+ if (verbose)
+ {
+ Info<< "after append: " << tok.info() << endl;
+ }
+ }
}
if (verbose)
{
Info<< nl;
IOobject::writeDivider(Info);
+
+ Info<< "tokenList:" << tokens << endl;
}
}
}
diff --git a/src/OpenFOAM/db/IOstreams/IOstreams/Istream.C b/src/OpenFOAM/db/IOstreams/IOstreams/Istream.C
index 25ee28e005..941b78cfd6 100644
--- a/src/OpenFOAM/db/IOstreams/IOstreams/Istream.C
+++ b/src/OpenFOAM/db/IOstreams/IOstreams/Istream.C
@@ -27,7 +27,7 @@ License
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
-void Foam::Istream::putBack(const token& t)
+void Foam::Istream::putBack(const token& tok)
{
if (bad())
{
@@ -43,13 +43,13 @@ void Foam::Istream::putBack(const token& t)
}
else
{
- putBackToken_ = t;
+ putBackToken_ = tok;
putBack_ = true;
}
}
-bool Foam::Istream::getBack(token& t)
+bool Foam::Istream::getBack(token& tok)
{
if (bad())
{
@@ -59,7 +59,7 @@ bool Foam::Istream::getBack(token& t)
}
else if (putBack_)
{
- t = putBackToken_;
+ tok = putBackToken_;
putBack_ = false;
return true;
}
@@ -68,15 +68,15 @@ bool Foam::Istream::getBack(token& t)
}
-bool Foam::Istream::peekBack(token& t)
+bool Foam::Istream::peekBack(token& tok)
{
if (putBack_)
{
- t = putBackToken_;
+ tok = putBackToken_;
}
else
{
- t = token::undefinedToken;
+ tok = token::undefinedToken;
}
return putBack_;
diff --git a/src/OpenFOAM/db/IOstreams/IOstreams/Istream.H b/src/OpenFOAM/db/IOstreams/IOstreams/Istream.H
index 24df823bdd..e55211309d 100644
--- a/src/OpenFOAM/db/IOstreams/IOstreams/Istream.H
+++ b/src/OpenFOAM/db/IOstreams/IOstreams/Istream.H
@@ -96,16 +96,16 @@ public:
//- Put back token
// Only a single put back is permitted
- void putBack(const token&);
+ void putBack(const token& tok);
//- Get the put back token if there is one and return true.
// Return false if no put back token is available.
- bool getBack(token&);
+ bool getBack(token& tok);
//- Peek at the put back token without removing it.
// Returns false if no put back token is available and set the
// token to undefined.
- bool peekBack(token&);
+ bool peekBack(token& tok);
//- Return next token from stream
virtual Istream& read(token&) = 0;
@@ -116,7 +116,7 @@ public:
//- Read a word
virtual Istream& read(word&) = 0;
- // Read a string (including enclosing double-quotes)
+ //- Read a string (including enclosing double-quotes)
virtual Istream& read(string&) = 0;
//- Read a label
diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UIPstream.C b/src/OpenFOAM/db/IOstreams/Pstreams/UIPstream.C
index 0e40098fa5..1ed965b758 100644
--- a/src/OpenFOAM/db/IOstreams/Pstreams/UIPstream.C
+++ b/src/OpenFOAM/db/IOstreams/Pstreams/UIPstream.C
@@ -3,7 +3,7 @@
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2015 OpenFOAM Foundation
- \\/ M anipulation |
+ \\/ M anipulation | Copyright (C) 2017 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@@ -164,22 +164,20 @@ Foam::Istream& Foam::UIPstream::read(token& t)
// Word
case token::tokenType::WORD :
{
- word* pval = new word;
- if (read(*pval))
+ word val;
+ if (read(val))
{
- if (token::compound::isCompound(*pval))
+ if (token::compound::isCompound(val))
{
- t = token::compound::New(*pval, *this).ptr();
- delete pval;
+ t = token::compound::New(val, *this).ptr();
}
else
{
- t = pval;
+ t = std::move(val);
}
}
else
{
- delete pval;
t.setBad();
}
return *this;
@@ -190,26 +188,25 @@ Foam::Istream& Foam::UIPstream::read(token& t)
{
// Recurse to read actual string
read(t);
- t.type() = token::tokenType::VERBATIMSTRING;
+ t.setType(token::tokenType::VERBATIMSTRING);
return *this;
}
case token::tokenType::VARIABLE :
{
// Recurse to read actual string
read(t);
- t.type() = token::tokenType::VARIABLE;
+ t.setType(token::tokenType::VARIABLE);
return *this;
}
case token::tokenType::STRING :
{
- string* pval = new string;
- if (read(*pval))
+ string val;
+ if (read(val))
{
- t = pval;
+ t = std::move(val);
}
else
{
- delete pval;
t.setBad();
}
return *this;
diff --git a/src/OpenFOAM/db/IOstreams/Sstreams/ISstream.C b/src/OpenFOAM/db/IOstreams/Sstreams/ISstream.C
index 010eef975b..cc9a0e5a65 100644
--- a/src/OpenFOAM/db/IOstreams/Sstreams/ISstream.C
+++ b/src/OpenFOAM/db/IOstreams/Sstreams/ISstream.C
@@ -116,21 +116,18 @@ char Foam::ISstream::nextValid()
void Foam::ISstream::readWordToken(token& t)
{
- word* wPtr = new word;
-
- if (read(*wPtr).bad())
+ word val;
+ if (read(val).bad())
{
- delete wPtr;
t.setBad();
}
- else if (token::compound::isCompound(*wPtr))
+ else if (token::compound::isCompound(val))
{
- t = token::compound::New(*wPtr, *this).ptr();
- delete wPtr;
+ t = token::compound::New(val, *this).ptr();
}
else
{
- t = wPtr; // Token takes ownership
+ t = std::move(val); // Move contents to token
}
}
@@ -192,16 +189,15 @@ Foam::Istream& Foam::ISstream::read(token& t)
case token::BEGIN_STRING :
{
putback(c);
- string* sPtr = new string;
- if (read(*sPtr).bad())
+ string val;
+ if (read(val).bad())
{
- delete sPtr;
t.setBad();
}
else
{
- t = sPtr; // Token takes ownership
+ t = std::move(val); // Move contents to token
}
return *this;
@@ -219,17 +215,16 @@ Foam::Istream& Foam::ISstream::read(token& t)
else if (nextC == token::BEGIN_BLOCK)
{
// Verbatim string: #{ ... #}
- string* sPtr = new string;
- if (readVerbatim(*sPtr).bad())
+ string val;
+ if (readVerbatim(val).bad())
{
- delete sPtr;
t.setBad();
}
else
{
- t = sPtr; // Token takes ownership
- t.type() = token::tokenType::VERBATIMSTRING;
+ t = std::move(val); // Move contents to token
+ t.setType(token::tokenType::VERBATIMSTRING);
}
}
else
@@ -259,17 +254,15 @@ Foam::Istream& Foam::ISstream::read(token& t)
putback(nextC);
putback(c);
- string* sPtr = new string;
-
- if (readVariable(*sPtr).bad())
+ string val;
+ if (readVariable(val).bad())
{
- delete sPtr;
t.setBad();
}
else
{
- t = sPtr; // Token takes ownership
- t.type() = token::tokenType::VARIABLE;
+ t = std::move(val); // Move contents to token
+ t.setType(token::tokenType::VARIABLE);
}
}
else
diff --git a/src/OpenFOAM/db/IOstreams/token/token.C b/src/OpenFOAM/db/IOstreams/token/token.C
index 17019bda99..fe1f76df5c 100644
--- a/src/OpenFOAM/db/IOstreams/token/token.C
+++ b/src/OpenFOAM/db/IOstreams/token/token.C
@@ -3,7 +3,7 @@
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2016 OpenFOAM Foundation
- \\/ M anipulation |
+ \\/ M anipulation | Copyright (C) 2017 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@@ -29,14 +29,14 @@ License
namespace Foam
{
- const char* const token::typeName = "token";
- token token::undefinedToken;
-
typedef token::compound tokenCompound;
defineTypeNameAndDebug(tokenCompound, 0);
defineRunTimeSelectionTable(tokenCompound, Istream);
}
+const char* const Foam::token::typeName = "token";
+const Foam::token Foam::token::undefinedToken;
+
// * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * * //
@@ -94,7 +94,7 @@ Foam::token::compound& Foam::token::transferCompoundToken(const Istream& is)
{
if (type_ == tokenType::COMPOUND)
{
- if (compoundTokenPtr_->empty())
+ if (data_.compoundPtr->empty())
{
FatalIOErrorInFunction(is)
<< "compound has already been transfered from token\n "
@@ -102,16 +102,14 @@ Foam::token::compound& Foam::token::transferCompoundToken(const Istream& is)
}
else
{
- compoundTokenPtr_->empty() = true;
+ data_.compoundPtr->empty() = true;
}
- return *compoundTokenPtr_;
- }
- else
- {
- parseError("compound");
- return *compoundTokenPtr_;
+ return *data_.compoundPtr;
}
+
+ parseError("compound");
+ return *data_.compoundPtr;
}
diff --git a/src/OpenFOAM/db/IOstreams/token/token.H b/src/OpenFOAM/db/IOstreams/token/token.H
index 1de16e5e13..90897917c3 100644
--- a/src/OpenFOAM/db/IOstreams/token/token.H
+++ b/src/OpenFOAM/db/IOstreams/token/token.H
@@ -3,7 +3,7 @@
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2016 OpenFOAM Foundation
- \\/ M anipulation |
+ \\/ M anipulation | Copyright (C) 2017 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@@ -25,7 +25,7 @@ Class
Foam::token
Description
- A token holds items read from Istream.
+ A token holds an item read from Istream.
SourceFiles
tokenI.H
@@ -64,34 +64,40 @@ Ostream& operator<<(Ostream& os, const token& t);
/*---------------------------------------------------------------------------*\
- Class token Declaration
+ Class token Declaration
\*---------------------------------------------------------------------------*/
class token
{
-
public:
- //- Enumeration defining the types of token
+ //- Enumeration defining the types of token.
+ // Since these values are also used to tag content in Pstream,
+ // the maximum number of types is limited to 30.
enum tokenType
{
- UNDEFINED,
+ UNDEFINED, //!< An undefined token-type
- PUNCTUATION,
- WORD,
- VARIABLE,
- STRING,
- VERBATIMSTRING,
- LABEL,
- FLOAT_SCALAR,
- DOUBLE_SCALAR,
- COMPOUND,
+ // Fundamental types
+ PUNCTUATION, //!< single character punctuation
+ LABEL, //!< label (integer) type
+ FLOAT_SCALAR, //!< float (single-precision) type
+ DOUBLE_SCALAR, //!< double (double-precision) type
- ERROR
+ // Pointer types
+ WORD, //!< Contents represent a Foam::word
+ STRING, //!< Contents represent a Foam::string
+ VARIABLE, //!< Contents are a Foam::string representing a
+ //!< dictionary \c $variable expansion
+ VERBATIMSTRING, //!< Contents are a Foam::string representing verbatim
+ //!< content
+ COMPOUND, //!< Compound type such as List\ etc.
+
+ ERROR //!< A token error encountered
};
- //- Standard punctuation tokens
+ //- Standard punctuation tokens (a character)
enum punctuationToken
{
NULL_TOKEN = '\0',
@@ -99,25 +105,26 @@ public:
TAB = '\t',
NL = '\n',
- END_STATEMENT = ';',
- BEGIN_LIST = '(',
- END_LIST = ')',
- BEGIN_SQR = '[',
- END_SQR = ']',
- BEGIN_BLOCK = '{',
- END_BLOCK = '}',
- COLON = ':',
- COMMA = ',',
+ END_STATEMENT = ';', //!< End entry [#isseparator]
+ BEGIN_LIST = '(', //!< Begin list [#isseparator]
+ END_LIST = ')', //!< End list [#isseparator]
+ BEGIN_SQR = '[', //!< Begin dimensions [#isseparator]
+ END_SQR = ']', //!< End dimensions [#isseparator]
+ BEGIN_BLOCK = '{', //!< Begin block [#isseparator]
+ END_BLOCK = '}', //!< End block [#isseparator]
+ COLON = ':', //!< Colon [#isseparator]
+ COMMA = ',', //!< Comma [#isseparator]
HASH = '#',
+ ATSYM = '@',
BEGIN_STRING = '"',
END_STRING = BEGIN_STRING,
- ASSIGN = '=',
- ADD = '+',
- SUBTRACT = '-',
- MULTIPLY = '*',
- DIVIDE = '/'
+ ASSIGN = '=', //!< Assigment/equals [#isseparator]
+ ADD = '+', //!< Addition [#isseparator]
+ SUBTRACT = '-', //!< Substract or start of negative number
+ MULTIPLY = '*', //!< Multiply [#isseparator]
+ DIVIDE = '/' //!< Divide [#isseparator]
};
@@ -145,7 +152,6 @@ public:
//- Runtime type information
TypeName("compound");
-
//- Declare run-time constructor selection table
declareRunTimeSelectionTable
(
@@ -169,7 +175,7 @@ public:
// Selectors
//- Select null constructed
- static autoPtr New(const word& type, Istream&);
+ static autoPtr New(const word& type, Istream& is);
//- Destructor
@@ -178,26 +184,22 @@ public:
// Member Functions
- // Access
+ //- Return true if name is a known compound type
+ static bool isCompound(const word& name);
- //- Return true if name is a compound type
- static bool isCompound(const word& name);
+ bool empty() const
+ {
+ return empty_;
+ }
- bool empty() const
- {
- return empty_;
- }
+ bool& empty()
+ {
+ return empty_;
+ }
- bool& empty()
- {
- return empty_;
- }
+ virtual label size() const = 0;
- virtual label size() const = 0;
-
- // Write
-
- virtual void write(Ostream& os) const = 0;
+ virtual void write(Ostream& os) const = 0;
// IOstream Operators
@@ -236,35 +238,49 @@ public:
//- Static undefined token
- static token undefinedToken;
+ static const token undefinedToken;
private:
+ //- A %union of token types
+ union content
+ {
+ // Fundamental values. Largest first for any {} initialization.
+ int64_t int64Val;
+ int32_t int32Val;
+
+ punctuationToken punctuationVal;
+ label labelVal;
+ floatScalar floatVal;
+ doubleScalar doubleVal;
+
+ // Pointers
+ word* wordPtr;
+ string* stringPtr;
+ mutable compound* compoundPtr;
+ };
+
+
// Private data
+ //- The data content (as a union).
+ // For memory alignment this should appear as the first member.
+ content data_;
+
//- The token type
tokenType type_;
- //- Anonymous Union of token types
- union
- {
- punctuationToken punctuationToken_;
- word* wordTokenPtr_;
- string* stringTokenPtr_;
- label labelToken_;
- floatScalar floatScalarToken_;
- doubleScalar doubleScalarToken_;
- mutable compound* compoundTokenPtr_;
- };
-
- //- Line number in the file this token was read from
+ //- Line number in the file the token was read from
label lineNumber_;
// Private Member Functions
- //- Clear any allocated storage (word or string)
+ //- Set as UNDEFINED and zero the union content without any checking
+ inline void setUndefined();
+
+ //- Clear any allocated storage (word or string) and set to UNDEFINED
inline void clear();
// Parse error, expected 'expected', found ...
@@ -283,18 +299,15 @@ public:
//- Construct null
inline token();
- //- Construct as copy
+ //- Copy construct
inline token(const token& t);
+ //- Move construct. The original token is left as UNDEFINED.
+ inline token(token&& t);
+
//- Construct punctuation character token
inline explicit token(punctuationToken p);
- //- Construct word token
- inline explicit token(const word& w);
-
- //- Construct string token
- inline explicit token(const string& str);
-
//- Construct label token
inline explicit token(const label val);
@@ -304,16 +317,16 @@ public:
//- Construct doubleScalar token
inline explicit token(const doubleScalar val);
+ //- Construct word token by copying word contents
+ inline explicit token(const word& w);
+
+ //- Construct string token by copying string contents
+ inline explicit token(const string& str);
+
//- Construct punctuation character token
inline token(punctuationToken p, const label lineNumber);
- //- Construct word token
- inline token(const word& w, const label lineNumber);
-
- //- Construct string token
- inline token(const string& str, const label lineNumber);
-
//- Construct label token
inline token(const label val, const label lineNumber);
@@ -323,6 +336,12 @@ public:
//- Construct doubleScalar token
inline token(const doubleScalar val, const label lineNumber);
+ //- Construct word token by copying word contents
+ inline token(const word& w, const label lineNumber);
+
+ //- Construct string token by copying string contents
+ inline token(const string& str, const label lineNumber);
+
//- Construct from Istream
token(Istream& is);
@@ -331,108 +350,216 @@ public:
inline ~token();
+ // Static member functions
+
+ //- True if the character is a punctuation separator (eg, in ISstream).
+ // Since it could also start a number, SUBTRACT is not included as
+ // a separator.
+ //
+ // \param c the character to test, passed as int for consistency with
+ // isdigit, isspace etc.
+ inline static bool isseparator(int c);
+
+
// Member functions
- // Access
+ // Status
- inline tokenType type() const;
- inline tokenType& type();
+ //- Return the name of the token type
+ word name() const;
- inline bool good() const;
- inline bool undefined() const;
- inline bool error() const;
+ //- Return the token type
+ inline tokenType type() const;
- inline bool isPunctuation() const;
- inline punctuationToken pToken() const;
+ //- Change the token type, for similar types.
+ // This can be used to change between string-like variants
+ // (eg, STRING, VARIABLE, VERBATIMSTRING)
+ // To change types entirely (eg, STRING to DOUBLE_SCALAR),
+ // use the corresponding assignment operator.
+ //
+ // \return true if the change was successful or no change was required
+ inline bool setType(const tokenType variant);
- inline bool isWord() const;
- inline const word& wordToken() const;
+ //- The line number for the token
+ inline label lineNumber() const;
- inline bool isVariable() const;
+ //- The line number for the token
+ inline label& lineNumber();
- inline bool isString() const;
- inline const string& stringToken() const;
+ //- True if token is not UNDEFINED or ERROR
+ inline bool good() const;
- inline bool isLabel() const;
- inline label labelToken() const;
+ //- True if token is UNDEFINED
+ inline bool undefined() const;
- inline bool isFloatScalar() const;
- inline floatScalar floatScalarToken() const;
+ //- True if token is ERROR
+ inline bool error() const;
- inline bool isDoubleScalar() const;
- inline doubleScalar doubleScalarToken() const;
+ //- True if token is PUNCTUATION
+ inline bool isPunctuation() const;
- inline bool isScalar() const;
- inline scalar scalarToken() const;
+ //- True if token is PUNCTUATION and isseparator
+ inline bool isSeparator() const;
- inline bool isNumber() const;
- inline scalar number() const;
+ //- True if token is LABEL
+ inline bool isLabel() const;
- inline bool isCompound() const;
- inline const compound& compoundToken() const;
- compound& transferCompoundToken(const Istream& is);
+ //- True if token is FLOAT_SCALAR
+ inline bool isFloatScalar() const;
- inline label lineNumber() const;
- inline label& lineNumber();
+ //- True if token is DOUBLE_SCALAR
+ inline bool isDoubleScalar() const;
+
+ //- True if token is FLOAT_SCALAR or DOUBLE_SCALAR
+ inline bool isScalar() const;
+
+ //- True if token is LABEL, FLOAT_SCALAR or DOUBLE_SCALAR
+ inline bool isNumber() const;
+
+ //- True if token is WORD
+ inline bool isWord() const;
+
+ //- True if token is STRING, VARIABLE or VERBATIMSTRING
+ inline bool isString() const;
+
+ //- True if token is VARIABLE
+ inline bool isVariable() const;
+
+ //- True if token is COMPOUND
+ inline bool isCompound() const;
- // Edit
+ // Access
- //- Set bad
- inline void setBad();
+ //- Return punctuation character.
+ // Report FatalIOError and return \b \\0 if token is not PUNCTUATION
+ inline punctuationToken pToken() const;
+
+ //- Return label value.
+ // Report FatalIOError and return \b 0 if token is not LABEL
+ inline label labelToken() const;
+
+ //- Return float value.
+ // Report FatalIOError and return \b 0.0 if token is not FLOAT_SCALAR
+ inline floatScalar floatScalarToken() const;
+
+ //- Return double value.
+ // Report FatalIOError and return \b 0.0 if token is not DOUBLE_SCALAR
+ inline doubleScalar doubleScalarToken() const;
+
+ //- Return float or double value.
+ // Report FatalIOError and return \b 0.0 if token is not a
+ // FLOAT_SCALAR or DOUBLE_SCALAR
+ inline scalar scalarToken() const;
+
+ //- Return label, float or double value.
+ // Report FatalIOError and return \b 0.0 if token is not a
+ // LABEL, FLOAT_SCALAR or DOUBLE_SCALAR
+ inline scalar number() const;
+
+ //- Return const reference to the word contents.
+ // Report FatalIOError and return \b "" if token is not a WORD
+ inline const word& wordToken() const;
+
+ //- Return const reference to the string contents.
+ // Report FatalIOError and return \b "" if token is not a
+ // STRING, VARIABLE or VERBATIMSTRING
+ inline const string& stringToken() const;
+
+ //- Read access for compound token
+ inline const compound& compoundToken() const;
+
+ //- Return reference to compound token and decrease its internal
+ //- refCound accordingly.
+ // The Istream is used for reference error messages only.
+ compound& transferCompoundToken(const Istream& is);
- // Info
+ // Edit
- //- Return info proxy.
- // Used to print token information to a stream
- InfoProxy info() const
- {
- return *this;
- }
+ //- Clear token and set to be in an error state.
+ inline void setBad();
+
+ //- Swap token contents: type, data, line-number
+ inline void swap(token& tok);
+
+
+ // Info
+
+ //- Return info proxy for printing token information to a stream
+ InfoProxy info() const
+ {
+ return *this;
+ }
// Member operators
- // Assignment
+ // Assignment
- inline void operator=(const token& t);
+ //- Copy assign
+ inline void operator=(const token& tok);
- inline void operator=(const punctuationToken p);
+ //- Move assign
+ inline void operator=(token&& tok);
- inline void operator=(word* wPtr);
- inline void operator=(const word& w);
+ //- Copy assign from punctuation
+ inline void operator=(const punctuationToken p);
- inline void operator=(string* strPtr);
- inline void operator=(const string& str);
+ //- Copy assign from label
+ inline void operator=(const label val);
- inline void operator=(const label val);
- inline void operator=(const floatScalar val);
- inline void operator=(const doubleScalar val);
+ //- Copy assign from float
+ inline void operator=(const floatScalar val);
- inline void operator=(compound* compPtr);
+ //- Copy assign from double
+ inline void operator=(const doubleScalar val);
+
+ //- Copy assign from word
+ inline void operator=(const word& w);
+
+ //- Copy assign from string
+ inline void operator=(const string& str);
+
+ //- Move assign from word
+ inline void operator=(word&& w);
+
+ //- Move assign from string
+ inline void operator=(string&& str);
+
+ //- Transfer word pointer to the token
+ // \deprecated in favour of using move assign from word
+ inline void operator=(word* wordPtr);
+
+ //- Transfer string pointer to the token
+ // \deprecated in favour of using move assign from string
+ inline void operator=(string* stringPtr);
+
+ //- Assign compound with reference counting to token
+ inline void operator=(compound* compoundPtr);
- // Equality
+ // Equality
- inline bool operator==(const token& t) const;
- inline bool operator==(const punctuationToken p) const;
- inline bool operator==(const word& w) const;
- inline bool operator==(const string& str) const;
- inline bool operator==(const label val) const;
- inline bool operator==(const floatScalar val) const;
- inline bool operator==(const doubleScalar val) const;
+ inline bool operator==(const token& t) const;
+
+ inline bool operator==(const punctuationToken p) const;
+ inline bool operator==(const label val) const;
+ inline bool operator==(const floatScalar val) const;
+ inline bool operator==(const doubleScalar val) const;
+ inline bool operator==(const word& w) const;
+ inline bool operator==(const string& str) const;
- // Inequality
+ // Inequality
- inline bool operator!=(const token& t) const;
- inline bool operator!=(const punctuationToken p) const;
- inline bool operator!=(const word& w) const;
- inline bool operator!=(const string& str) const;
- inline bool operator!=(const label val) const;
- inline bool operator!=(const floatScalar val) const;
- inline bool operator!=(const doubleScalar val) const;
+ inline bool operator!=(const token& t) const;
+ inline bool operator!=(const punctuationToken p) const;
+ inline bool operator!=(const label val) const;
+ inline bool operator!=(const floatScalar val) const;
+ inline bool operator!=(const doubleScalar val) const;
+ inline bool operator!=(const word& w) const;
+ inline bool operator!=(const string& str) const;
// IOstream operators
diff --git a/src/OpenFOAM/db/IOstreams/token/tokenI.H b/src/OpenFOAM/db/IOstreams/token/tokenI.H
index 32417b4e75..c391298652 100644
--- a/src/OpenFOAM/db/IOstreams/token/tokenI.H
+++ b/src/OpenFOAM/db/IOstreams/token/tokenI.H
@@ -3,7 +3,7 @@
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2016 OpenFOAM Foundation
- \\/ M anipulation |
+ \\/ M anipulation | Copyright (C) 2017 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@@ -23,36 +23,54 @@ License
\*---------------------------------------------------------------------------*/
+#include
+
// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
+inline void Foam::token::setUndefined()
+{
+ type_ = tokenType::UNDEFINED;
+ data_.int64Val = 0; // bit-wise zero for union content
+ // leave lineNumber untouched - may still be needed
+}
+
+
inline void Foam::token::clear()
{
- if (type_ == tokenType::WORD)
+ switch (type_)
{
- delete wordTokenPtr_;
- }
- else if
- (
- type_ == tokenType::STRING
- || type_ == tokenType::VARIABLE
- || type_ == tokenType::VERBATIMSTRING
- )
- {
- delete stringTokenPtr_;
- }
- else if (type_ == tokenType::COMPOUND)
- {
- if (compoundTokenPtr_->unique())
+ case tokenType::WORD:
{
- delete compoundTokenPtr_;
+ delete data_.wordPtr;
+ break;
}
- else
+
+ case tokenType::STRING:
+ case tokenType::VARIABLE:
+ case tokenType::VERBATIMSTRING:
{
- compoundTokenPtr_->refCount::operator--();
+ delete data_.stringPtr;
+ break;
}
+
+ case tokenType::COMPOUND:
+ {
+ if (data_.compoundPtr->unique())
+ {
+ delete data_.compoundPtr;
+ }
+ else
+ {
+ data_.compoundPtr->refCount::operator--();
+ }
+ break;
+ }
+
+ default:
+ break;
}
- type_ = tokenType::UNDEFINED;
+ setUndefined();
}
@@ -60,58 +78,62 @@ inline void Foam::token::clear()
inline Foam::token::token()
:
+ data_(), // bit-wise zero for union content
type_(tokenType::UNDEFINED),
lineNumber_(0)
{}
-inline Foam::token::token(const token& t)
+inline Foam::token::token(const token& tok)
:
- type_(t.type_),
- lineNumber_(t.lineNumber_)
+ data_(tok.data_), // bit-wise copy of union content
+ type_(tok.type_),
+ lineNumber_(tok.lineNumber_)
{
+ // Fundamental: values already handled by bit-wise copy
+ // Pointer: duplicate content or increase refCount
+
switch (type_)
{
- case tokenType::UNDEFINED:
- break;
-
- case tokenType::PUNCTUATION:
- punctuationToken_ = t.punctuationToken_;
- break;
-
case tokenType::WORD:
- wordTokenPtr_ = new word(*t.wordTokenPtr_);
- break;
+ {
+ data_.wordPtr = new word(*tok.data_.wordPtr);
+ break;
+ }
case tokenType::STRING:
case tokenType::VARIABLE:
case tokenType::VERBATIMSTRING:
- stringTokenPtr_ = new string(*t.stringTokenPtr_);
- break;
-
- case tokenType::LABEL:
- labelToken_ = t.labelToken_;
- break;
-
- case tokenType::FLOAT_SCALAR:
- floatScalarToken_ = t.floatScalarToken_;
- break;
-
- case tokenType::DOUBLE_SCALAR:
- doubleScalarToken_ = t.doubleScalarToken_;
- break;
+ {
+ data_.stringPtr = new string(*tok.data_.stringPtr);
+ break;
+ }
case tokenType::COMPOUND:
- compoundTokenPtr_ = t.compoundTokenPtr_;
- compoundTokenPtr_->refCount::operator++();
- break;
+ {
+ // Identical pointers, but increase the refCount
+ data_.compoundPtr = tok.data_.compoundPtr;
+ data_.compoundPtr->refCount::operator++();
+ break;
+ }
- case tokenType::ERROR:
- break;
+ default:
+ break;
}
}
+inline Foam::token::token(token&& tok)
+:
+ data_(tok.data_), // bit-wise copy of union content
+ type_(tok.type_),
+ lineNumber_(tok.lineNumber_)
+{
+ tok.setUndefined(); // zero the union content without any checking
+ tok.lineNumber_ = 0;
+}
+
+
inline Foam::token::token(punctuationToken p)
:
token(p, 0)
@@ -150,50 +172,62 @@ inline Foam::token::token(const doubleScalar val)
inline Foam::token::token(punctuationToken p, label lineNumber)
:
+ data_(),
type_(tokenType::PUNCTUATION),
- punctuationToken_(p),
lineNumber_(lineNumber)
-{}
-
-
-inline Foam::token::token(const word& w, label lineNumber)
-:
- type_(tokenType::WORD),
- wordTokenPtr_(new word(w)),
- lineNumber_(lineNumber)
-{}
-
-
-inline Foam::token::token(const string& str, label lineNumber)
-:
- type_(tokenType::STRING),
- stringTokenPtr_(new string(str)),
- lineNumber_(lineNumber)
-{}
+{
+ data_.punctuationVal = p;
+}
inline Foam::token::token(const label val, label lineNumber)
:
+ data_(),
type_(tokenType::LABEL),
- labelToken_(val),
lineNumber_(lineNumber)
-{}
+{
+ data_.labelVal = val;
+}
inline Foam::token::token(const floatScalar val, label lineNumber)
:
+ data_(),
type_(tokenType::FLOAT_SCALAR),
- floatScalarToken_(val),
lineNumber_(lineNumber)
-{}
+{
+ data_.floatVal = val;
+}
inline Foam::token::token(const doubleScalar val, label lineNumber)
:
+ data_(),
type_(tokenType::DOUBLE_SCALAR),
- doubleScalarToken_(val),
lineNumber_(lineNumber)
-{}
+{
+ data_.doubleVal = val;
+}
+
+
+inline Foam::token::token(const word& w, label lineNumber)
+:
+ data_(),
+ type_(tokenType::WORD),
+ lineNumber_(lineNumber)
+{
+ data_.wordPtr = new word(w);
+}
+
+
+inline Foam::token::token(const string& str, label lineNumber)
+:
+ data_(),
+ type_(tokenType::STRING),
+ lineNumber_(lineNumber)
+{
+ data_.stringPtr = new string(str);
+}
// * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * //
@@ -206,134 +240,179 @@ inline Foam::token::~token()
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
-inline Foam::token::tokenType Foam::token::type() const
+inline void Foam::token::swap(token& tok)
+{
+ std::swap(data_, tok.data_);
+ std::swap(type_, tok.type_);
+ std::swap(lineNumber_, tok.lineNumber_);
+}
+
+
+inline Foam::token::tokenType Foam::token::type() const
{
return type_;
}
-inline Foam::token::tokenType& Foam::token::type()
+
+inline bool Foam::token::setType(token::tokenType variant)
{
- return type_;
+ if (type_ == variant)
+ {
+ // No change required
+ return true;
+ }
+
+ switch (variant)
+ {
+ case tokenType::STRING:
+ case tokenType::VARIABLE:
+ case tokenType::VERBATIMSTRING:
+ {
+ switch (type_)
+ {
+ // could also go from WORD to STRING etc - to be decided
+ case tokenType::STRING:
+ case tokenType::VARIABLE:
+ case tokenType::VERBATIMSTRING:
+ type_ = variant;
+ return true;
+ break;
+
+ default:
+ return false;
+ break;
+ }
+ }
+
+ default:
+ break;
+ }
+
+ return false;
}
+
+inline Foam::label Foam::token::lineNumber() const
+{
+ return lineNumber_;
+}
+
+
+inline Foam::label& Foam::token::lineNumber()
+{
+ return lineNumber_;
+}
+
+
inline bool Foam::token::good() const
{
- return (type_ != tokenType::ERROR && type_ != tokenType::UNDEFINED);
+ return (type_ != tokenType::UNDEFINED && type_ != tokenType::ERROR);
}
+
inline bool Foam::token::undefined() const
{
return (type_ == tokenType::UNDEFINED);
}
+
inline bool Foam::token::error() const
{
return (type_ == tokenType::ERROR);
}
+
inline bool Foam::token::isPunctuation() const
{
return (type_ == tokenType::PUNCTUATION);
}
-inline Foam::token::punctuationToken Foam::token::pToken() const
+
+inline Foam::token::punctuationToken Foam::token::pToken() const
{
if (type_ == tokenType::PUNCTUATION)
{
- return punctuationToken_;
- }
- else
- {
- parseError("punctuation character");
- return NULL_TOKEN;
+ return data_.punctuationVal;
}
+
+ parseError("punctuation character");
+ return NULL_TOKEN;
}
-inline bool Foam::token::isWord() const
+
+inline bool Foam::token::isseparator(int c)
{
- return (type_ == tokenType::WORD);
-}
-
-inline const Foam::word& Foam::token::wordToken() const
-{
- if (type_ == tokenType::WORD)
+ switch (c)
{
- return *wordTokenPtr_;
- }
- else
- {
- parseError(word::typeName);
- return word::null;
+ case token::END_STATEMENT :
+ case token::BEGIN_LIST :
+ case token::END_LIST :
+ case token::BEGIN_SQR :
+ case token::END_SQR :
+ case token::BEGIN_BLOCK :
+ case token::END_BLOCK :
+ case token::COLON :
+ case token::COMMA :
+ case token::ASSIGN :
+ case token::ADD :
+ // Excluded token::SUBTRACT since it could start a number
+ case token::MULTIPLY :
+ case token::DIVIDE :
+ {
+ return true;
+ }
+
+ default:
+ break;
}
+
+ return false;
}
-inline bool Foam::token::isVariable() const
-{
- return (type_ == tokenType::VARIABLE);
-}
-inline bool Foam::token::isString() const
+inline bool Foam::token::isSeparator() const
{
return
(
- type_ == tokenType::STRING
- || type_ == tokenType::VARIABLE
- || type_ == tokenType::VERBATIMSTRING
+ type_ == tokenType::PUNCTUATION
+ && isseparator(data_.punctuationVal)
);
}
-inline const Foam::string& Foam::token::stringToken() const
-{
- if
- (
- type_ == tokenType::STRING
- || type_ == tokenType::VARIABLE
- || type_ == tokenType::VERBATIMSTRING
- )
- {
- return *stringTokenPtr_;
- }
- else
- {
- parseError(string::typeName);
- return string::null;
- }
-}
inline bool Foam::token::isLabel() const
{
return (type_ == tokenType::LABEL);
}
+
inline Foam::label Foam::token::labelToken() const
{
if (type_ == tokenType::LABEL)
{
- return labelToken_;
- }
- else
- {
- parseError(pTraits