ENH: enhancements to behaviour of token

- improved memory alignment reduces overhead for Int32 compilation

- added move/swap semantics

- made the type() readonly in favour of setVariant() to allow change
  of variant within a particular storage representation.
  Eg, STRING -> VERBATIMSTRING.

applications/test/token/Make/files

@@ -0,0 +1,3 @@
+Test-token.C
+
+EXE = $(FOAM_USER_APPBIN)/Test-token

applications/test/token/Make/options

@@ -0,0 +1 @@
+/* EXE_INC = */

applications/test/token/Test-token.C

@@ -0,0 +1,75 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | Copyright (C) 2017 OpenCFD Ltd.
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+Description
+    Test token construct assign etc.
+\*---------------------------------------------------------------------------*/
+
+#include "argList.H"
+#include "IOobject.H"
+#include "IOstreams.H"
+#include "IFstream.H"
+#include "StringStream.H"
+#include "cpuTime.H"
+#include "DynamicList.H"
+
+using namespace Foam;
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+//  Main program:
+
+int main(int argc, char *argv[])
+{
+    argList::noBanner();
+    argList::noParallel();
+
+    argList args(argc, argv, false, true);
+
+    token tok1;
+    Info<< "construct null: " << tok1.info() << endl;
+
+    tok1 = double(3.14159);
+    Info<< "assign double: " << tok1.info() << endl;
+
+    token tok2(tok1);
+    Info<< "copy construct: " << tok2.info() << endl;
+
+    tok1 = word("this-word");
+    Info<< "assign word: " << tok1.info() << endl;
+
+    token tok3(tok1);
+    Info<< "copy construct: " << tok3.info() << endl;
+    Info<< "orig: " << tok1.info() << endl;
+
+    token tok4(std::move(tok1));
+    Info<< "move construct: " << tok4.info() << endl;
+    Info<< "orig: " << tok1.info() << endl;
+
+    tok3 = tok4;
+    Info<< "assign token: " << tok3.info() << endl;
+    Info<< "orig: " << tok4.info() << endl;
+
+    return 0;
+}
+
+// ************************************************************************* //

applications/test/tokenize/Test-tokenize.C

@@ -33,6 +33,7 @@ Description
 #include "IFstream.H"
 #include "StringStream.H"
 #include "cpuTime.H"
+#include "DynamicList.H"
 
 using namespace Foam;
 
@@ -69,6 +70,8 @@ int main(int argc, char *argv[])
 
             IStringStream is(rawArg);
 
+            DynamicList<token> tokens;
+
             while (is.good())
             {
                 token tok(is);
@@ -83,12 +86,23 @@ int main(int argc, char *argv[])
                         << "  lookahead: '" << char(lookahead) << "'"
                         << endl;
                 }
+
+                if (tok.good())
+                {
+                    tokens.append(std::move(tok));
+                    if (verbose)
+                    {
+                        Info<< "after append: " << tok.info() << endl;
+                    }
+                }
             }
 
             if (verbose)
             {
                 Info<< nl;
                 IOobject::writeDivider(Info);
+
+                Info<< "tokenList:" << tokens << endl;
             }
         }
     }

src/OpenFOAM/db/IOstreams/IOstreams/Istream.C

@@ -27,7 +27,7 @@ License
 
 // * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
 
-void Foam::Istream::putBack(const token& t)
+void Foam::Istream::putBack(const token& tok)
 {
     if (bad())
     {
@@ -43,13 +43,13 @@ void Foam::Istream::putBack(const token& t)
     }
     else
     {
-        putBackToken_ = t;
+        putBackToken_ = tok;
         putBack_ = true;
     }
 }
 
 
-bool Foam::Istream::getBack(token& t)
+bool Foam::Istream::getBack(token& tok)
 {
     if (bad())
     {
@@ -59,7 +59,7 @@ bool Foam::Istream::getBack(token& t)
     }
     else if (putBack_)
     {
-        t = putBackToken_;
+        tok = putBackToken_;
         putBack_ = false;
         return true;
     }
@@ -68,15 +68,15 @@ bool Foam::Istream::getBack(token& t)
 }
 
 
-bool Foam::Istream::peekBack(token& t)
+bool Foam::Istream::peekBack(token& tok)
 {
     if (putBack_)
     {
-        t = putBackToken_;
+        tok = putBackToken_;
     }
     else
     {
-        t = token::undefinedToken;
+        tok = token::undefinedToken;
     }
 
     return putBack_;

src/OpenFOAM/db/IOstreams/IOstreams/Istream.H

@@ -96,16 +96,16 @@ public:
 
             //- Put back token
             //  Only a single put back is permitted
-            void putBack(const token&);
+            void putBack(const token& tok);
 
             //- Get the put back token if there is one and return true.
             //  Return false if no put back token is available.
-            bool getBack(token&);
+            bool getBack(token& tok);
 
             //- Peek at the put back token without removing it.
             //  Returns false if no put back token is available and set the
             //  token to undefined.
-            bool peekBack(token&);
+            bool peekBack(token& tok);
 
             //- Return next token from stream
             virtual Istream& read(token&) = 0;
@@ -116,7 +116,7 @@ public:
             //- Read a word
             virtual Istream& read(word&) = 0;
 
-            // Read a string (including enclosing double-quotes)
+            //- Read a string (including enclosing double-quotes)
             virtual Istream& read(string&) = 0;
 
             //- Read a label

src/OpenFOAM/db/IOstreams/Pstreams/UIPstream.C

@@ -3,7 +3,7 @@
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
     \\  /    A nd           | Copyright (C) 2011-2015 OpenFOAM Foundation
-     \\/     M anipulation  |
+     \\/     M anipulation  | Copyright (C) 2017 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -164,22 +164,20 @@ Foam::Istream& Foam::UIPstream::read(token& t)
         // Word
         case token::tokenType::WORD :
         {
-            word* pval = new word;
+            word val;
-            if (read(*pval))
+            if (read(val))
             {
-                if (token::compound::isCompound(*pval))
+                if (token::compound::isCompound(val))
                 {
-                    t = token::compound::New(*pval, *this).ptr();
+                    t = token::compound::New(val, *this).ptr();
-                    delete pval;
                 }
                 else
                 {
-                    t = pval;
+                    t = std::move(val);
                 }
             }
             else
             {
-                delete pval;
                 t.setBad();
             }
             return *this;
@@ -190,26 +188,25 @@ Foam::Istream& Foam::UIPstream::read(token& t)
         {
             // Recurse to read actual string
             read(t);
-            t.type() = token::tokenType::VERBATIMSTRING;
+            t.setType(token::tokenType::VERBATIMSTRING);
             return *this;
         }
         case token::tokenType::VARIABLE :
         {
             // Recurse to read actual string
             read(t);
-            t.type() = token::tokenType::VARIABLE;
+            t.setType(token::tokenType::VARIABLE);
             return *this;
         }
         case token::tokenType::STRING :
         {
-            string* pval = new string;
+            string val;
-            if (read(*pval))
+            if (read(val))
             {
-                t = pval;
+                t = std::move(val);
             }
             else
             {
-                delete pval;
                 t.setBad();
             }
             return *this;

src/OpenFOAM/db/IOstreams/Sstreams/ISstream.C

@@ -116,21 +116,18 @@ char Foam::ISstream::nextValid()
 
 void Foam::ISstream::readWordToken(token& t)
 {
-    word* wPtr = new word;
+    word val;
-
+    if (read(val).bad())
-    if (read(*wPtr).bad())
     {
-        delete wPtr;
         t.setBad();
     }
-    else if (token::compound::isCompound(*wPtr))
+    else if (token::compound::isCompound(val))
     {
-        t = token::compound::New(*wPtr, *this).ptr();
+        t = token::compound::New(val, *this).ptr();
-        delete wPtr;
     }
     else
     {
-        t = wPtr; // Token takes ownership
+        t = std::move(val); // Move contents to token
     }
 }
 
@@ -192,16 +189,15 @@ Foam::Istream& Foam::ISstream::read(token& t)
         case token::BEGIN_STRING :
         {
             putback(c);
-            string* sPtr = new string;
 
-            if (read(*sPtr).bad())
+            string val;
+            if (read(val).bad())
             {
-                delete sPtr;
                 t.setBad();
             }
             else
             {
-                t = sPtr; // Token takes ownership
+                t = std::move(val); // Move contents to token
             }
 
             return *this;
@@ -219,17 +215,16 @@ Foam::Istream& Foam::ISstream::read(token& t)
             else if (nextC == token::BEGIN_BLOCK)
             {
                 // Verbatim string: #{ ... #}
-                string* sPtr = new string;
 
-                if (readVerbatim(*sPtr).bad())
+                string val;
+                if (readVerbatim(val).bad())
                 {
-                    delete sPtr;
                     t.setBad();
                 }
                 else
                 {
-                    t = sPtr; // Token takes ownership
+                    t = std::move(val); // Move contents to token
-                    t.type() = token::tokenType::VERBATIMSTRING;
+                    t.setType(token::tokenType::VERBATIMSTRING);
                 }
             }
             else
@@ -259,17 +254,15 @@ Foam::Istream& Foam::ISstream::read(token& t)
                 putback(nextC);
                 putback(c);
 
-                string* sPtr = new string;
+                string val;
-
+                if (readVariable(val).bad())
-                if (readVariable(*sPtr).bad())
                 {
-                    delete sPtr;
                     t.setBad();
                 }
                 else
                 {
-                    t = sPtr; // Token takes ownership
+                    t = std::move(val); // Move contents to token
-                    t.type() = token::tokenType::VARIABLE;
+                    t.setType(token::tokenType::VARIABLE);
                 }
             }
             else

src/OpenFOAM/db/IOstreams/token/token.C

@@ -3,7 +3,7 @@
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
     \\  /    A nd           | Copyright (C) 2011-2016 OpenFOAM Foundation
-     \\/     M anipulation  |
+     \\/     M anipulation  | Copyright (C) 2017 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -29,14 +29,14 @@ License
 
 namespace Foam
 {
-    const char* const token::typeName = "token";
-    token token::undefinedToken;
-
     typedef token::compound tokenCompound;
     defineTypeNameAndDebug(tokenCompound, 0);
     defineRunTimeSelectionTable(tokenCompound, Istream);
 }
 
+const char* const Foam::token::typeName = "token";
+const Foam::token Foam::token::undefinedToken;
+
 
 // * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * * //
 
@@ -94,7 +94,7 @@ Foam::token::compound& Foam::token::transferCompoundToken(const Istream& is)
 {
     if (type_ == tokenType::COMPOUND)
     {
-        if (compoundTokenPtr_->empty())
+        if (data_.compoundPtr->empty())
         {
             FatalIOErrorInFunction(is)
                 << "compound has already been transfered from token\n    "
@@ -102,16 +102,14 @@ Foam::token::compound& Foam::token::transferCompoundToken(const Istream& is)
         }
         else
         {
-            compoundTokenPtr_->empty() = true;
+            data_.compoundPtr->empty() = true;
         }
 
-        return *compoundTokenPtr_;
+        return *data_.compoundPtr;
-    }
-    else
-    {
-        parseError("compound");
-        return *compoundTokenPtr_;
     }
+
+    parseError("compound");
+    return *data_.compoundPtr;
 }
 
 

src/OpenFOAM/db/IOstreams/token/token.H

@@ -3,7 +3,7 @@
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
     \\  /    A nd           | Copyright (C) 2011-2016 OpenFOAM Foundation
-     \\/     M anipulation  |
+     \\/     M anipulation  | Copyright (C) 2017 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -25,7 +25,7 @@ Class
     Foam::token
 
 Description
-    A token holds items read from Istream.
+    A token holds an item read from Istream.
 
 SourceFiles
     tokenI.H
@@ -64,34 +64,40 @@ Ostream& operator<<(Ostream& os, const token& t);
 
 
 /*---------------------------------------------------------------------------*\
-                           Class token Declaration
+                            Class token Declaration
 \*---------------------------------------------------------------------------*/
 
 class token
 {
-
 public:
 
-    //- Enumeration defining the types of token
+    //- Enumeration defining the types of token.
+    //  Since these values are also used to tag content in Pstream,
+    //  the maximum number of types is limited to 30.
     enum tokenType
     {
-        UNDEFINED,
+        UNDEFINED,        //!< An undefined token-type
 
-        PUNCTUATION,
+        // Fundamental types
-        WORD,
+        PUNCTUATION,      //!< single character punctuation
-        VARIABLE,
+        LABEL,            //!< label (integer) type
-        STRING,
+        FLOAT_SCALAR,     //!< float (single-precision) type
-        VERBATIMSTRING,
+        DOUBLE_SCALAR,    //!< double (double-precision) type
-        LABEL,
-        FLOAT_SCALAR,
-        DOUBLE_SCALAR,
-        COMPOUND,
 
-        ERROR
+        // Pointer types
+        WORD,             //!< Contents represent a Foam::word
+        STRING,           //!< Contents represent a Foam::string
+        VARIABLE,         //!< Contents are a Foam::string representing a
+                          //!< dictionary \c $variable expansion
+        VERBATIMSTRING,   //!< Contents are a Foam::string representing verbatim
+                          //!< content
+        COMPOUND,         //!< Compound type such as List\<label\> etc.
+
+        ERROR             //!< A token error encountered
     };
 
 
-    //- Standard punctuation tokens
+    //- Standard punctuation tokens (a character)
     enum punctuationToken
     {
         NULL_TOKEN     = '\0',
@@ -99,25 +105,26 @@ public:
         TAB            = '\t',
         NL             = '\n',
 
-        END_STATEMENT  = ';',
+        END_STATEMENT  = ';', //!< End entry [#isseparator]
-        BEGIN_LIST     = '(',
+        BEGIN_LIST     = '(', //!< Begin list [#isseparator]
-        END_LIST       = ')',
+        END_LIST       = ')', //!< End list [#isseparator]
-        BEGIN_SQR      = '[',
+        BEGIN_SQR      = '[', //!< Begin dimensions [#isseparator]
-        END_SQR        = ']',
+        END_SQR        = ']', //!< End dimensions [#isseparator]
-        BEGIN_BLOCK    = '{',
+        BEGIN_BLOCK    = '{', //!< Begin block [#isseparator]
-        END_BLOCK      = '}',
+        END_BLOCK      = '}', //!< End block [#isseparator]
-        COLON          = ':',
+        COLON          = ':', //!< Colon [#isseparator]
-        COMMA          = ',',
+        COMMA          = ',', //!< Comma [#isseparator]
         HASH           = '#',
+        ATSYM          = '@',
 
         BEGIN_STRING   = '"',
         END_STRING     = BEGIN_STRING,
 
-        ASSIGN         = '=',
+        ASSIGN         = '=', //!< Assigment/equals [#isseparator]
-        ADD            = '+',
+        ADD            = '+', //!< Addition [#isseparator]
-        SUBTRACT       = '-',
+        SUBTRACT       = '-', //!< Substract or start of negative number
-        MULTIPLY       = '*',
+        MULTIPLY       = '*', //!< Multiply [#isseparator]
-        DIVIDE         = '/'
+        DIVIDE         = '/'  //!< Divide [#isseparator]
     };
 
 
@@ -145,7 +152,6 @@ public:
         //- Runtime type information
         TypeName("compound");
 
-
         //- Declare run-time constructor selection table
         declareRunTimeSelectionTable
         (
@@ -169,7 +175,7 @@ public:
         // Selectors
 
             //- Select null constructed
-            static autoPtr<compound> New(const word& type, Istream&);
+            static autoPtr<compound> New(const word& type, Istream& is);
 
 
         //- Destructor
@@ -178,26 +184,22 @@ public:
 
         // Member Functions
 
-            // Access
+            //- Return true if name is a known compound type
+            static bool isCompound(const word& name);
 
-                //- Return true if name is a compound type
+            bool empty() const
-                static bool isCompound(const word& name);
+            {
+                return empty_;
+            }
 
-                bool empty() const
+            bool& empty()
-                {
+            {
-                    return empty_;
+                return empty_;
-                }
+            }
 
-                bool& empty()
+            virtual label size() const = 0;
-                {
-                    return empty_;
-                }
 
-                virtual label size() const = 0;
+            virtual void write(Ostream& os) const = 0;
-
-            // Write
-
-                virtual void write(Ostream& os) const = 0;
 
 
         // IOstream Operators
@@ -236,35 +238,49 @@ public:
 
 
     //- Static undefined token
-    static token undefinedToken;
+    static const token undefinedToken;
 
 
 private:
 
+    //- A %union of token types
+    union content
+    {
+        // Fundamental values. Largest first for any {} initialization.
+        int64_t int64Val;
+        int32_t int32Val;
+
+        punctuationToken punctuationVal;
+        label labelVal;
+        floatScalar floatVal;
+        doubleScalar doubleVal;
+
+        // Pointers
+        word* wordPtr;
+        string* stringPtr;
+        mutable compound* compoundPtr;
+    };
+
+
     // Private data
 
+        //- The data content (as a union).
+        //  For memory alignment this should appear as the first member.
+        content data_;
+
         //- The token type
         tokenType type_;
 
-        //- Anonymous Union of token types
+        //- Line number in the file the token was read from
-        union
-        {
-            punctuationToken punctuationToken_;
-            word* wordTokenPtr_;
-            string* stringTokenPtr_;
-            label labelToken_;
-            floatScalar floatScalarToken_;
-            doubleScalar doubleScalarToken_;
-            mutable compound* compoundTokenPtr_;
-        };
-
-        //- Line number in the file this token was read from
         label lineNumber_;
 
 
     // Private Member Functions
 
-        //- Clear any allocated storage (word or string)
+        //- Set as UNDEFINED and zero the union content without any checking
+        inline void setUndefined();
+
+        //- Clear any allocated storage (word or string) and set to UNDEFINED
         inline void clear();
 
         // Parse error, expected 'expected', found ...
@@ -283,18 +299,15 @@ public:
         //- Construct null
         inline token();
 
-        //- Construct as copy
+        //- Copy construct
         inline token(const token& t);
 
+        //- Move construct. The original token is left as UNDEFINED.
+        inline token(token&& t);
+
         //- Construct punctuation character token
         inline explicit token(punctuationToken p);
 
-        //- Construct word token
-        inline explicit token(const word& w);
-
-        //- Construct string token
-        inline explicit token(const string& str);
-
         //- Construct label token
         inline explicit token(const label val);
 
@@ -304,16 +317,16 @@ public:
         //- Construct doubleScalar token
         inline explicit token(const doubleScalar val);
 
+        //- Construct word token by copying word contents
+        inline explicit token(const word& w);
+
+        //- Construct string token by copying string contents
+        inline explicit token(const string& str);
+
 
         //- Construct punctuation character token
         inline token(punctuationToken p, const label lineNumber);
 
-        //- Construct word token
-        inline token(const word& w, const label lineNumber);
-
-        //- Construct string token
-        inline token(const string& str, const label lineNumber);
-
         //- Construct label token
         inline token(const label val, const label lineNumber);
 
@@ -323,6 +336,12 @@ public:
         //- Construct doubleScalar token
         inline token(const doubleScalar val, const label lineNumber);
 
+        //- Construct word token by copying word contents
+        inline token(const word& w, const label lineNumber);
+
+        //- Construct string token by copying string contents
+        inline token(const string& str, const label lineNumber);
+
         //- Construct from Istream
         token(Istream& is);
 
@@ -331,108 +350,216 @@ public:
     inline ~token();
 
 
+    // Static member functions
+
+        //- True if the character is a punctuation separator (eg, in ISstream).
+        //  Since it could also start a number, SUBTRACT is not included as
+        //  a separator.
+        //
+        //  \param c the character to test, passed as int for consistency with
+        //      isdigit, isspace etc.
+        inline static bool isseparator(int c);
+
+
     // Member functions
 
-        // Access
+    // Status
 
-            inline tokenType type() const;
+        //- Return the name of the token type
-            inline tokenType& type();
+        word name() const;
 
-            inline bool good() const;
+        //- Return the token type
-            inline bool undefined() const;
+        inline tokenType type() const;
-            inline bool error() const;
 
-            inline bool isPunctuation() const;
+        //- Change the token type, for similar types.
-            inline punctuationToken pToken() const;
+        //  This can be used to change between string-like variants
+        //  (eg, STRING, VARIABLE, VERBATIMSTRING)
+        //  To change types entirely (eg, STRING to DOUBLE_SCALAR),
+        //  use the corresponding assignment operator.
+        //
+        //  \return true if the change was successful or no change was required
+        inline bool setType(const tokenType variant);
 
-            inline bool isWord() const;
+        //- The line number for the token
-            inline const word& wordToken() const;
+        inline label lineNumber() const;
 
-            inline bool isVariable() const;
+        //- The line number for the token
+        inline label& lineNumber();
 
-            inline bool isString() const;
+        //- True if token is not UNDEFINED or ERROR
-            inline const string& stringToken() const;
+        inline bool good() const;
 
-            inline bool isLabel() const;
+        //- True if token is UNDEFINED
-            inline label labelToken() const;
+        inline bool undefined() const;
 
-            inline bool isFloatScalar() const;
+        //- True if token is ERROR
-            inline floatScalar floatScalarToken() const;
+        inline bool error() const;
 
-            inline bool isDoubleScalar() const;
+        //- True if token is PUNCTUATION
-            inline doubleScalar doubleScalarToken() const;
+        inline bool isPunctuation() const;
 
-            inline bool isScalar() const;
+        //- True if token is PUNCTUATION and isseparator
-            inline scalar scalarToken() const;
+        inline bool isSeparator() const;
 
-            inline bool isNumber() const;
+        //- True if token is LABEL
-            inline scalar number() const;
+        inline bool isLabel() const;
 
-            inline bool isCompound() const;
+        //- True if token is FLOAT_SCALAR
-            inline const compound& compoundToken() const;
+        inline bool isFloatScalar() const;
-            compound& transferCompoundToken(const Istream& is);
 
-            inline label lineNumber() const;
+        //- True if token is DOUBLE_SCALAR
-            inline label& lineNumber();
+        inline bool isDoubleScalar() const;
+
+        //- True if token is FLOAT_SCALAR or DOUBLE_SCALAR
+        inline bool isScalar() const;
+
+        //- True if token is LABEL, FLOAT_SCALAR or DOUBLE_SCALAR
+        inline bool isNumber() const;
+
+        //- True if token is WORD
+        inline bool isWord() const;
+
+        //- True if token is STRING, VARIABLE or VERBATIMSTRING
+        inline bool isString() const;
+
+        //- True if token is VARIABLE
+        inline bool isVariable() const;
+
+        //- True if token is COMPOUND
+        inline bool isCompound() const;
 
 
-        // Edit
+    // Access
 
-            //- Set bad
+        //- Return punctuation character.
-            inline void setBad();
+        //  Report FatalIOError and return \b \\0 if token is not PUNCTUATION
+        inline punctuationToken pToken() const;
+
+        //- Return label value.
+        //  Report FatalIOError and return \b 0 if token is not LABEL
+        inline label labelToken() const;
+
+        //- Return float value.
+        //  Report FatalIOError and return \b 0.0 if token is not FLOAT_SCALAR
+        inline floatScalar floatScalarToken() const;
+
+        //- Return double value.
+        //  Report FatalIOError and return \b 0.0 if token is not DOUBLE_SCALAR
+        inline doubleScalar doubleScalarToken() const;
+
+        //- Return float or double value.
+        //  Report FatalIOError and return \b 0.0 if token is not a
+        //  FLOAT_SCALAR or DOUBLE_SCALAR
+        inline scalar scalarToken() const;
+
+        //- Return label, float or double value.
+        //  Report FatalIOError and return \b 0.0 if token is not a
+        //  LABEL, FLOAT_SCALAR or DOUBLE_SCALAR
+        inline scalar number() const;
+
+        //- Return const reference to the word contents.
+        //  Report FatalIOError and return \b "" if token is not a WORD
+        inline const word& wordToken() const;
+
+        //- Return const reference to the string contents.
+        //  Report FatalIOError and return \b "" if token is not a
+        //  STRING, VARIABLE or VERBATIMSTRING
+        inline const string& stringToken() const;
+
+        //- Read access for compound token
+        inline const compound& compoundToken() const;
+
+        //- Return reference to compound token and decrease its internal
+        //- refCound accordingly.
+        //  The Istream is used for reference error messages only.
+        compound& transferCompoundToken(const Istream& is);
 
 
-        // Info
+    // Edit
 
-            //- Return info proxy.
+        //- Clear token and set to be in an error state.
-            //  Used to print token information to a stream
+        inline void setBad();
-            InfoProxy<token> info() const
+
-            {
+        //- Swap token contents: type, data, line-number
-                return *this;
+        inline void swap(token& tok);
-            }
+
+
+    // Info
+
+        //- Return info proxy for printing token information to a stream
+        InfoProxy<token> info() const
+        {
+            return *this;
+        }
 
 
     // Member operators
 
-        // Assignment
+    // Assignment
 
-            inline void operator=(const token& t);
+        //- Copy assign
+        inline void operator=(const token& tok);
 
-            inline void operator=(const punctuationToken p);
+        //- Move assign
+        inline void operator=(token&& tok);
 
-            inline void operator=(word* wPtr);
+        //- Copy assign from punctuation
-            inline void operator=(const word& w);
+        inline void operator=(const punctuationToken p);
 
-            inline void operator=(string* strPtr);
+        //- Copy assign from label
-            inline void operator=(const string& str);
+        inline void operator=(const label val);
 
-            inline void operator=(const label val);
+        //- Copy assign from float
-            inline void operator=(const floatScalar val);
+        inline void operator=(const floatScalar val);
-            inline void operator=(const doubleScalar val);
 
-            inline void operator=(compound* compPtr);
+        //- Copy assign from double
+        inline void operator=(const doubleScalar val);
+
+        //- Copy assign from word
+        inline void operator=(const word& w);
+
+        //- Copy assign from string
+        inline void operator=(const string& str);
+
+        //- Move assign from word
+        inline void operator=(word&& w);
+
+        //- Move assign from string
+        inline void operator=(string&& str);
+
+        //- Transfer word pointer to the token
+        //  \deprecated in favour of using move assign from word
+        inline void operator=(word* wordPtr);
+
+        //- Transfer string pointer to the token
+        //  \deprecated in favour of using move assign from string
+        inline void operator=(string* stringPtr);
+
+        //- Assign compound with reference counting to token
+        inline void operator=(compound* compoundPtr);
 
 
-        // Equality
+    // Equality
 
-            inline bool operator==(const token& t) const;
+        inline bool operator==(const token& t) const;
-            inline bool operator==(const punctuationToken p) const;
+
-            inline bool operator==(const word& w) const;
+        inline bool operator==(const punctuationToken p) const;
-            inline bool operator==(const string& str) const;
+        inline bool operator==(const label val) const;
-            inline bool operator==(const label val) const;
+        inline bool operator==(const floatScalar val) const;
-            inline bool operator==(const floatScalar val) const;
+        inline bool operator==(const doubleScalar val) const;
-            inline bool operator==(const doubleScalar val) const;
+        inline bool operator==(const word& w) const;
+        inline bool operator==(const string& str) const;
 
 
-        // Inequality
+    // Inequality
 
-            inline bool operator!=(const token& t) const;
+        inline bool operator!=(const token& t) const;
-            inline bool operator!=(const punctuationToken p) const;
+        inline bool operator!=(const punctuationToken p) const;
-            inline bool operator!=(const word& w) const;
+        inline bool operator!=(const label val) const;
-            inline bool operator!=(const string& str) const;
+        inline bool operator!=(const floatScalar val) const;
-            inline bool operator!=(const label val) const;
+        inline bool operator!=(const doubleScalar val) const;
-            inline bool operator!=(const floatScalar val) const;
+        inline bool operator!=(const word& w) const;
-            inline bool operator!=(const doubleScalar val) const;
+        inline bool operator!=(const string& str) const;
 
 
     // IOstream operators

src/OpenFOAM/db/IOstreams/token/tokenIO.C

@@ -3,7 +3,7 @@
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
     \\  /    A nd           | Copyright (C) 2011-2015 OpenFOAM Foundation
-     \\/     M anipulation  |
+     \\/     M anipulation  | Copyright (C) 2017 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -25,32 +25,129 @@ License
 
 #include "error.H"
 #include "token.H"
-
-#include "IOstreams.H"
 #include "scalar.H"
+#include "IOstreams.H"
+
+// * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+template<class OS>
+static OS& printTokenInfo(OS& os, const token& tok)
+{
+    os  << "on line " << tok.lineNumber() << ": ";
+
+    switch (tok.type())
+    {
+        case token::tokenType::UNDEFINED:
+            os  << "undefined token";
+        break;
+
+        case token::tokenType::PUNCTUATION:
+            os  << "punctuation '" << tok.pToken() << '\'';
+        break;
+
+        case token::tokenType::LABEL:
+            os  << "label " << tok.labelToken();
+        break;
+
+        case token::tokenType::FLOAT_SCALAR:
+            os  << "float " << tok.floatScalarToken();
+        break;
+
+        case token::tokenType::DOUBLE_SCALAR:
+            os  << "double " << tok.doubleScalarToken();
+        break;
+
+        case token::tokenType::WORD:
+            os  << "word '" << tok.wordToken() << '\'';
+        break;
+
+        case token::tokenType::STRING:
+            os  << "string " << tok.stringToken();
+        break;
+
+        case token::tokenType::VARIABLE:
+            os  << "variable " << tok.stringToken();
+        break;
+
+        case token::tokenType::VERBATIMSTRING:
+            os  << "verbatim string " << tok.stringToken();
+        break;
+
+        case token::tokenType::COMPOUND:
+        {
+            if (tok.compoundToken().empty())
+            {
+                os  << "empty ";
+            }
+            os  << "compound of type "
+                << tok.compoundToken().type();
+        }
+        break;
+
+        case token::tokenType::ERROR:
+            os  << "error";
+        break;
+
+        default:
+            os  << "unknown token type '" << int(tok.type()) << '\'';
+            break;
+    }
+
+    return os;
+}
+} // End namespace Foam
+
 
 // * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
 
 Foam::token::token(Istream& is)
 :
-    type_(tokenType::UNDEFINED)
+    token()
 {
     is.read(*this);
 }
 
 
-// * * * * * * * * * * * * * * * IOstream Operators  * * * * * * * * * * * * //
+// * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
 
-Foam::Istream& Foam::operator>>(Istream& is, token& t)
+Foam::word Foam::token::name() const
 {
-    t.clear();
+    switch (type_)
-    return is.read(t);
+    {
+        case token::tokenType::UNDEFINED: return "undefined";
+        case token::tokenType::PUNCTUATION: return "punctuation";
+        case token::tokenType::LABEL: return "label";
+        case token::tokenType::FLOAT_SCALAR: return "float";
+        case token::tokenType::DOUBLE_SCALAR: return "double";
+        case token::tokenType::WORD: return "word";
+        case token::tokenType::STRING: return "string";
+        case token::tokenType::VERBATIMSTRING: return "verbatim";
+        case token::tokenType::VARIABLE: return "variable";
+        case token::tokenType::COMPOUND: return "compound";
+        case token::tokenType::ERROR: return "error";
+
+        default:
+            break;
+    }
+
+    return "unknown(" + std::to_string(int(type_)) + ")";
 }
 
 
-Foam::Ostream& Foam::operator<<(Ostream& os, const token& t)
+// * * * * * * * * * * * * * * * IOstream Operators  * * * * * * * * * * * * //
+
+Foam::Istream& Foam::operator>>(Istream& is, token& tok)
 {
-    switch (t.type_)
+    tok.clear();
+    return is.read(tok);
+}
+
+
+Foam::Ostream& Foam::operator<<(Ostream& os, const token& tok)
+{
+    switch (tok.type_)
     {
         case token::tokenType::UNDEFINED:
             os << "UNDEFINED";
@@ -59,37 +156,37 @@ Foam::Ostream& Foam::operator<<(Ostream& os, const token& t)
         break;
 
         case token::tokenType::PUNCTUATION:
-            os << t.punctuationToken_;
+            os << tok.data_.punctuationVal;
+        break;
+
+        case token::tokenType::LABEL:
+            os << tok.data_.labelVal;
+        break;
+
+        case token::tokenType::FLOAT_SCALAR:
+            os << tok.data_.floatVal;
+        break;
+
+        case token::tokenType::DOUBLE_SCALAR:
+            os << tok.data_.doubleVal;
         break;
 
         case token::tokenType::WORD:
-            os << *t.wordTokenPtr_;
+            os << *tok.data_.wordPtr;
         break;
 
         case token::tokenType::STRING:
         case token::tokenType::VERBATIMSTRING:
-            os << *t.stringTokenPtr_;
+            os << *tok.data_.stringPtr;
         break;
 
         case token::tokenType::VARIABLE:
             // Behaviour differs according to stream type
-            os.write(t);
+            os.write(tok);
-        break;
-
-        case token::tokenType::LABEL:
-            os << t.labelToken_;
-        break;
-
-        case token::tokenType::FLOAT_SCALAR:
-            os << t.floatScalarToken_;
-        break;
-
-        case token::tokenType::DOUBLE_SCALAR:
-            os << t.doubleScalarToken_;
         break;
 
         case token::tokenType::COMPOUND:
-            os << *t.compoundTokenPtr_;
+            os << *tok.data_.compoundPtr;
         break;
 
         case token::tokenType::ERROR:
@@ -135,144 +232,14 @@ Foam::Ostream& Foam::operator<<(Ostream& os, const token::compound& ct)
 
 ostream& Foam::operator<<(ostream& os, const InfoProxy<token>& ip)
 {
-    const token& t = ip.t_;
+    return printTokenInfo(os, ip.t_);
-
-    os  << "on line " << t.lineNumber();
-
-    switch (t.type())
-    {
-        case token::tokenType::UNDEFINED:
-            os  << " an undefined token";
-        break;
-
-        case token::tokenType::PUNCTUATION:
-            os  << " the punctuation token " << '\'' << t.pToken() << '\'';
-        break;
-
-        case token::tokenType::WORD:
-            os  << " the word " << '\'' << t.wordToken() << '\'';
-        break;
-
-        case token::tokenType::STRING:
-            os  << " the string " << t.stringToken();
-        break;
-
-        case token::tokenType::VARIABLE:
-            os  << " the variable " << t.stringToken();
-        break;
-
-        case token::tokenType::VERBATIMSTRING:
-            os  << " the verbatim string " << t.stringToken();
-        break;
-
-        case token::tokenType::LABEL:
-            os  << " the label " << t.labelToken();
-        break;
-
-        case token::tokenType::FLOAT_SCALAR:
-            os  << " the floatScalar " << t.floatScalarToken();
-        break;
-
-        case token::tokenType::DOUBLE_SCALAR:
-            os  << " the doubleScalar " << t.doubleScalarToken();
-        break;
-
-        case token::tokenType::COMPOUND:
-        {
-            if (t.compoundToken().empty())
-            {
-                os  << " the empty compound of type "
-                    << t.compoundToken().type();
-            }
-            else
-            {
-                os  << " the compound of type "
-                    << t.compoundToken().type();
-            }
-        }
-        break;
-
-        case token::tokenType::ERROR:
-            os  << " an error";
-        break;
-
-        default:
-            os  << " an unknown token type " << '\'' << int(t.type()) << '\'';
-    }
-
-    return os;
 }
 
 
 template<>
 Foam::Ostream& Foam::operator<<(Ostream& os, const InfoProxy<token>& ip)
 {
-    const token& t = ip.t_;
+    return printTokenInfo(os, ip.t_);
-
-    os  << "on line " << t.lineNumber();
-
-    switch (t.type())
-    {
-        case token::tokenType::UNDEFINED:
-            os  << " an undefined token";
-        break;
-
-        case token::tokenType::PUNCTUATION:
-            os  << " the punctuation token " << '\'' << t.pToken() << '\'';
-        break;
-
-        case token::tokenType::WORD:
-            os  << " the word " << '\'' << t.wordToken() << '\'';
-        break;
-
-        case token::tokenType::STRING:
-            os  << " the string " << t.stringToken();
-        break;
-
-        case token::tokenType::VARIABLE:
-            os  << " the variable " << t.stringToken();
-        break;
-
-        case token::tokenType::VERBATIMSTRING:
-            os  << " the verbatim string " << t.stringToken();
-        break;
-
-        case token::tokenType::LABEL:
-            os  << " the label " << t.labelToken();
-        break;
-
-        case token::tokenType::FLOAT_SCALAR:
-            os  << " the floatScalar " << t.floatScalarToken();
-        break;
-
-        case token::tokenType::DOUBLE_SCALAR:
-            os  << " the doubleScalar " << t.doubleScalarToken();
-        break;
-
-        case token::tokenType::COMPOUND:
-        {
-            if (t.compoundToken().empty())
-            {
-                os  << " the empty compound of type "
-                    << t.compoundToken().type();
-            }
-            else
-            {
-                os  << " the compound of type "
-                    << t.compoundToken().type();
-            }
-        }
-        break;
-
-        case token::tokenType::ERROR:
-            os  << " an error";
-        break;
-
-        default:
-            os  << " an unknown token type "  << '\'' << int(t.type()) << '\'';
-    }
-
-    return os;
 }
 
 

src/OpenFOAM/memory/refCount/refCount.H

@@ -36,8 +36,6 @@ See also
 #ifndef refCount_H
 #define refCount_H
 
-#include "bool.H"
-
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
 namespace Foam
@@ -50,20 +48,10 @@ namespace Foam
 class refCount
 {
     // Private data
-
+    int count_;
-        int count_;
 
 
-    // Private Member Functions
+public:
-
-        //- Dissallow copy
-        refCount(const refCount&);
-
-        //- Dissallow bitwise assignment
-        void operator=(const refCount&);
-
-
-protected:
 
     // Constructors
 
@@ -74,8 +62,6 @@ protected:
         {}
 
 
-public:
-
     // Member Functions
 
         //- Return the current reference count
@@ -87,7 +73,7 @@ public:
         //- Return true if the reference count is zero
         bool unique() const
         {
-            return count_ == 0;
+            return !count_;
         }
 
 
@@ -96,25 +82,25 @@ public:
         //- Increment the reference count
         void operator++()
         {
-            count_++;
+            ++count_;
         }
 
         //- Increment the reference count
         void operator++(int)
         {
-            count_++;
+            ++count_;
         }
 
         //- Decrement the reference count
         void operator--()
         {
-            count_--;
+            --count_;
         }
 
         //- Decrement the reference count
         void operator--(int)
         {
-            count_--;
+            --count_;
         }
 };
 

src/OpenFOAM/memory/tmp/tmp.H

@@ -161,7 +161,7 @@ public:
         //- Assignment to pointer changing this tmp to a temporary T
         inline void operator=(T* tPtr);
 
-        //- Assignment transfering the temporary T to this tmp
+        //- Assignment transferring the temporary T to this tmp
         inline void operator=(const tmp<T>& t);
 };