ENH: enhancements to behaviour of token

- improved memory alignment reduces overhead for Int32 compilation

- added move/swap semantics

- made the type() readonly in favour of setVariant() to allow change
  of variant within a particular storage representation.
  Eg, STRING -> VERBATIMSTRING.

applications/test/token/Make/files

@@ -0,0 +1,3 @@
+Test-token.C
+
+EXE = $(FOAM_USER_APPBIN)/Test-token

applications/test/token/Make/options

@@ -0,0 +1 @@
+/* EXE_INC = */

applications/test/token/Test-token.C

@@ -0,0 +1,75 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | Copyright (C) 2017 OpenCFD Ltd.
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+Description
+    Test token construct assign etc.
+\*---------------------------------------------------------------------------*/
+
+#include "argList.H"
+#include "IOobject.H"
+#include "IOstreams.H"
+#include "IFstream.H"
+#include "StringStream.H"
+#include "cpuTime.H"
+#include "DynamicList.H"
+
+using namespace Foam;
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+//  Main program:
+
+int main(int argc, char *argv[])
+{
+    argList::noBanner();
+    argList::noParallel();
+
+    argList args(argc, argv, false, true);
+
+    token tok1;
+    Info<< "construct null: " << tok1.info() << endl;
+
+    tok1 = double(3.14159);
+    Info<< "assign double: " << tok1.info() << endl;
+
+    token tok2(tok1);
+    Info<< "copy construct: " << tok2.info() << endl;
+
+    tok1 = word("this-word");
+    Info<< "assign word: " << tok1.info() << endl;
+
+    token tok3(tok1);
+    Info<< "copy construct: " << tok3.info() << endl;
+    Info<< "orig: " << tok1.info() << endl;
+
+    token tok4(std::move(tok1));
+    Info<< "move construct: " << tok4.info() << endl;
+    Info<< "orig: " << tok1.info() << endl;
+
+    tok3 = tok4;
+    Info<< "assign token: " << tok3.info() << endl;
+    Info<< "orig: " << tok4.info() << endl;
+
+    return 0;
+}
+
+// ************************************************************************* //

applications/test/tokenize/Test-tokenize.C

@@ -33,6 +33,7 @@ Description
 #include "IFstream.H"
 #include "StringStream.H"
 #include "cpuTime.H"
+#include "DynamicList.H"
 
 using namespace Foam;
 
@@ -69,6 +70,8 @@ int main(int argc, char *argv[])
 
             IStringStream is(rawArg);
 
+            DynamicList<token> tokens;
+
             while (is.good())
             {
                 token tok(is);
@@ -83,12 +86,23 @@ int main(int argc, char *argv[])
                         << "  lookahead: '" << char(lookahead) << "'"
                         << endl;
                 }
+
+                if (tok.good())
+                {
+                    tokens.append(std::move(tok));
+                    if (verbose)
+                    {
+                        Info<< "after append: " << tok.info() << endl;
+                    }
+                }
             }
 
             if (verbose)
             {
                 Info<< nl;
                 IOobject::writeDivider(Info);
+
+                Info<< "tokenList:" << tokens << endl;
             }
         }
     }

src/OpenFOAM/db/IOstreams/IOstreams/Istream.C

@@ -27,7 +27,7 @@ License
 
 // * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
 
-void Foam::Istream::putBack(const token& t)
+void Foam::Istream::putBack(const token& tok)
 {
     if (bad())
     {
@@ -43,13 +43,13 @@ void Foam::Istream::putBack(const token& t)
     }
     else
     {
-        putBackToken_ = t;
+        putBackToken_ = tok;
         putBack_ = true;
     }
 }
 
 
-bool Foam::Istream::getBack(token& t)
+bool Foam::Istream::getBack(token& tok)
 {
     if (bad())
     {
@@ -59,7 +59,7 @@ bool Foam::Istream::getBack(token& t)
     }
     else if (putBack_)
     {
-        t = putBackToken_;
+        tok = putBackToken_;
         putBack_ = false;
         return true;
     }
@@ -68,15 +68,15 @@ bool Foam::Istream::getBack(token& t)
 }
 
 
-bool Foam::Istream::peekBack(token& t)
+bool Foam::Istream::peekBack(token& tok)
 {
     if (putBack_)
     {
-        t = putBackToken_;
+        tok = putBackToken_;
     }
     else
     {
-        t = token::undefinedToken;
+        tok = token::undefinedToken;
     }
 
     return putBack_;

src/OpenFOAM/db/IOstreams/IOstreams/Istream.H

@@ -96,16 +96,16 @@ public:
 
             //- Put back token
             //  Only a single put back is permitted
-            void putBack(const token&);
+            void putBack(const token& tok);
 
             //- Get the put back token if there is one and return true.
             //  Return false if no put back token is available.
-            bool getBack(token&);
+            bool getBack(token& tok);
 
             //- Peek at the put back token without removing it.
             //  Returns false if no put back token is available and set the
             //  token to undefined.
-            bool peekBack(token&);
+            bool peekBack(token& tok);
 
             //- Return next token from stream
             virtual Istream& read(token&) = 0;
@@ -116,7 +116,7 @@ public:
             //- Read a word
             virtual Istream& read(word&) = 0;
 
-            // Read a string (including enclosing double-quotes)
+            //- Read a string (including enclosing double-quotes)
             virtual Istream& read(string&) = 0;
 
             //- Read a label

src/OpenFOAM/db/IOstreams/Pstreams/UIPstream.C

@@ -3,7 +3,7 @@
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
     \\  /    A nd           | Copyright (C) 2011-2015 OpenFOAM Foundation
-     \\/     M anipulation  |
+     \\/     M anipulation  | Copyright (C) 2017 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -164,22 +164,20 @@ Foam::Istream& Foam::UIPstream::read(token& t)
         // Word
         case token::tokenType::WORD :
         {
-            word* pval = new word;
+            word val;
-            if (read(*pval))
+            if (read(val))
             {
-                if (token::compound::isCompound(*pval))
+                if (token::compound::isCompound(val))
                 {
-                    t = token::compound::New(*pval, *this).ptr();
+                    t = token::compound::New(val, *this).ptr();
-                    delete pval;
                 }
                 else
                 {
-                    t = pval;
+                    t = std::move(val);
                 }
             }
             else
             {
-                delete pval;
                 t.setBad();
             }
             return *this;
@@ -190,26 +188,25 @@ Foam::Istream& Foam::UIPstream::read(token& t)
         {
             // Recurse to read actual string
             read(t);
-            t.type() = token::tokenType::VERBATIMSTRING;
+            t.setType(token::tokenType::VERBATIMSTRING);
             return *this;
         }
         case token::tokenType::VARIABLE :
         {
             // Recurse to read actual string
             read(t);
-            t.type() = token::tokenType::VARIABLE;
+            t.setType(token::tokenType::VARIABLE);
             return *this;
         }
         case token::tokenType::STRING :
         {
-            string* pval = new string;
+            string val;
-            if (read(*pval))
+            if (read(val))
             {
-                t = pval;
+                t = std::move(val);
             }
             else
             {
-                delete pval;
                 t.setBad();
             }
             return *this;

src/OpenFOAM/db/IOstreams/Sstreams/ISstream.C

@@ -116,21 +116,18 @@ char Foam::ISstream::nextValid()
 
 void Foam::ISstream::readWordToken(token& t)
 {
-    word* wPtr = new word;
+    word val;
-
+    if (read(val).bad())
-    if (read(*wPtr).bad())
     {
-        delete wPtr;
         t.setBad();
     }
-    else if (token::compound::isCompound(*wPtr))
+    else if (token::compound::isCompound(val))
     {
-        t = token::compound::New(*wPtr, *this).ptr();
+        t = token::compound::New(val, *this).ptr();
-        delete wPtr;
     }
     else
     {
-        t = wPtr; // Token takes ownership
+        t = std::move(val); // Move contents to token
     }
 }
 
@@ -192,16 +189,15 @@ Foam::Istream& Foam::ISstream::read(token& t)
         case token::BEGIN_STRING :
         {
             putback(c);
-            string* sPtr = new string;
 
-            if (read(*sPtr).bad())
+            string val;
+            if (read(val).bad())
             {
-                delete sPtr;
                 t.setBad();
             }
             else
             {
-                t = sPtr; // Token takes ownership
+                t = std::move(val); // Move contents to token
             }
 
             return *this;
@@ -219,17 +215,16 @@ Foam::Istream& Foam::ISstream::read(token& t)
             else if (nextC == token::BEGIN_BLOCK)
             {
                 // Verbatim string: #{ ... #}
-                string* sPtr = new string;
 
-                if (readVerbatim(*sPtr).bad())
+                string val;
+                if (readVerbatim(val).bad())
                 {
-                    delete sPtr;
                     t.setBad();
                 }
                 else
                 {
-                    t = sPtr; // Token takes ownership
+                    t = std::move(val); // Move contents to token
-                    t.type() = token::tokenType::VERBATIMSTRING;
+                    t.setType(token::tokenType::VERBATIMSTRING);
                 }
             }
             else
@@ -259,17 +254,15 @@ Foam::Istream& Foam::ISstream::read(token& t)
                 putback(nextC);
                 putback(c);
 
-                string* sPtr = new string;
+                string val;
-
+                if (readVariable(val).bad())
-                if (readVariable(*sPtr).bad())
                 {
-                    delete sPtr;
                     t.setBad();
                 }
                 else
                 {
-                    t = sPtr; // Token takes ownership
+                    t = std::move(val); // Move contents to token
-                    t.type() = token::tokenType::VARIABLE;
+                    t.setType(token::tokenType::VARIABLE);
                 }
             }
             else

src/OpenFOAM/db/IOstreams/token/token.C

@@ -3,7 +3,7 @@
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
     \\  /    A nd           | Copyright (C) 2011-2016 OpenFOAM Foundation
-     \\/     M anipulation  |
+     \\/     M anipulation  | Copyright (C) 2017 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -29,14 +29,14 @@ License
 
 namespace Foam
 {
-    const char* const token::typeName = "token";
-    token token::undefinedToken;
-
     typedef token::compound tokenCompound;
     defineTypeNameAndDebug(tokenCompound, 0);
     defineRunTimeSelectionTable(tokenCompound, Istream);
 }
 
+const char* const Foam::token::typeName = "token";
+const Foam::token Foam::token::undefinedToken;
+
 
 // * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * * //
 
@@ -94,7 +94,7 @@ Foam::token::compound& Foam::token::transferCompoundToken(const Istream& is)
 {
     if (type_ == tokenType::COMPOUND)
     {
-        if (compoundTokenPtr_->empty())
+        if (data_.compoundPtr->empty())
         {
             FatalIOErrorInFunction(is)
                 << "compound has already been transfered from token\n    "
@@ -102,16 +102,14 @@ Foam::token::compound& Foam::token::transferCompoundToken(const Istream& is)
         }
         else
         {
-            compoundTokenPtr_->empty() = true;
+            data_.compoundPtr->empty() = true;
         }
 
-        return *compoundTokenPtr_;
+        return *data_.compoundPtr;
     }
-    else
+
-    {
     parseError("compound");
-        return *compoundTokenPtr_;
+    return *data_.compoundPtr;
-    }
 }
 
 

src/OpenFOAM/db/IOstreams/token/token.H

@@ -3,7 +3,7 @@
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
     \\  /    A nd           | Copyright (C) 2011-2016 OpenFOAM Foundation
-     \\/     M anipulation  |
+     \\/     M anipulation  | Copyright (C) 2017 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -25,7 +25,7 @@ Class
     Foam::token
 
 Description
-    A token holds items read from Istream.
+    A token holds an item read from Istream.
 
 SourceFiles
     tokenI.H
@@ -69,29 +69,35 @@ Ostream& operator<<(Ostream& os, const token& t);
 
 class token
 {
-
 public:
 
-    //- Enumeration defining the types of token
+    //- Enumeration defining the types of token.
+    //  Since these values are also used to tag content in Pstream,
+    //  the maximum number of types is limited to 30.
     enum tokenType
     {
-        UNDEFINED,
+        UNDEFINED,        //!< An undefined token-type
 
-        PUNCTUATION,
+        // Fundamental types
-        WORD,
+        PUNCTUATION,      //!< single character punctuation
-        VARIABLE,
+        LABEL,            //!< label (integer) type
-        STRING,
+        FLOAT_SCALAR,     //!< float (single-precision) type
-        VERBATIMSTRING,
+        DOUBLE_SCALAR,    //!< double (double-precision) type
-        LABEL,
-        FLOAT_SCALAR,
-        DOUBLE_SCALAR,
-        COMPOUND,
 
-        ERROR
+        // Pointer types
+        WORD,             //!< Contents represent a Foam::word
+        STRING,           //!< Contents represent a Foam::string
+        VARIABLE,         //!< Contents are a Foam::string representing a
+                          //!< dictionary \c $variable expansion
+        VERBATIMSTRING,   //!< Contents are a Foam::string representing verbatim
+                          //!< content
+        COMPOUND,         //!< Compound type such as List\<label\> etc.
+
+        ERROR             //!< A token error encountered
     };
 
 
-    //- Standard punctuation tokens
+    //- Standard punctuation tokens (a character)
     enum punctuationToken
     {
         NULL_TOKEN     = '\0',
@@ -99,25 +105,26 @@ public:
         TAB            = '\t',
         NL             = '\n',
 
-        END_STATEMENT  = ';',
+        END_STATEMENT  = ';', //!< End entry [#isseparator]
-        BEGIN_LIST     = '(',
+        BEGIN_LIST     = '(', //!< Begin list [#isseparator]
-        END_LIST       = ')',
+        END_LIST       = ')', //!< End list [#isseparator]
-        BEGIN_SQR      = '[',
+        BEGIN_SQR      = '[', //!< Begin dimensions [#isseparator]
-        END_SQR        = ']',
+        END_SQR        = ']', //!< End dimensions [#isseparator]
-        BEGIN_BLOCK    = '{',
+        BEGIN_BLOCK    = '{', //!< Begin block [#isseparator]
-        END_BLOCK      = '}',
+        END_BLOCK      = '}', //!< End block [#isseparator]
-        COLON          = ':',
+        COLON          = ':', //!< Colon [#isseparator]
-        COMMA          = ',',
+        COMMA          = ',', //!< Comma [#isseparator]
         HASH           = '#',
+        ATSYM          = '@',
 
         BEGIN_STRING   = '"',
         END_STRING     = BEGIN_STRING,
 
-        ASSIGN         = '=',
+        ASSIGN         = '=', //!< Assigment/equals [#isseparator]
-        ADD            = '+',
+        ADD            = '+', //!< Addition [#isseparator]
-        SUBTRACT       = '-',
+        SUBTRACT       = '-', //!< Substract or start of negative number
-        MULTIPLY       = '*',
+        MULTIPLY       = '*', //!< Multiply [#isseparator]
-        DIVIDE         = '/'
+        DIVIDE         = '/'  //!< Divide [#isseparator]
     };
 
 
@@ -145,7 +152,6 @@ public:
         //- Runtime type information
         TypeName("compound");
 
-
         //- Declare run-time constructor selection table
         declareRunTimeSelectionTable
         (
@@ -169,7 +175,7 @@ public:
         // Selectors
 
             //- Select null constructed
-            static autoPtr<compound> New(const word& type, Istream&);
+            static autoPtr<compound> New(const word& type, Istream& is);
 
 
         //- Destructor
@@ -178,9 +184,7 @@ public:
 
         // Member Functions
 
-            // Access
+            //- Return true if name is a known compound type
-
-                //- Return true if name is a compound type
             static bool isCompound(const word& name);
 
             bool empty() const
@@ -195,8 +199,6 @@ public:
 
             virtual label size() const = 0;
 
-            // Write
-
             virtual void write(Ostream& os) const = 0;
 
 
@@ -236,35 +238,49 @@ public:
 
 
     //- Static undefined token
-    static token undefinedToken;
+    static const token undefinedToken;
 
 
 private:
 
+    //- A %union of token types
+    union content
+    {
+        // Fundamental values. Largest first for any {} initialization.
+        int64_t int64Val;
+        int32_t int32Val;
+
+        punctuationToken punctuationVal;
+        label labelVal;
+        floatScalar floatVal;
+        doubleScalar doubleVal;
+
+        // Pointers
+        word* wordPtr;
+        string* stringPtr;
+        mutable compound* compoundPtr;
+    };
+
+
     // Private data
 
+        //- The data content (as a union).
+        //  For memory alignment this should appear as the first member.
+        content data_;
+
         //- The token type
         tokenType type_;
 
-        //- Anonymous Union of token types
+        //- Line number in the file the token was read from
-        union
-        {
-            punctuationToken punctuationToken_;
-            word* wordTokenPtr_;
-            string* stringTokenPtr_;
-            label labelToken_;
-            floatScalar floatScalarToken_;
-            doubleScalar doubleScalarToken_;
-            mutable compound* compoundTokenPtr_;
-        };
-
-        //- Line number in the file this token was read from
         label lineNumber_;
 
 
     // Private Member Functions
 
-        //- Clear any allocated storage (word or string)
+        //- Set as UNDEFINED and zero the union content without any checking
+        inline void setUndefined();
+
+        //- Clear any allocated storage (word or string) and set to UNDEFINED
         inline void clear();
 
         // Parse error, expected 'expected', found ...
@@ -283,18 +299,15 @@ public:
         //- Construct null
         inline token();
 
-        //- Construct as copy
+        //- Copy construct
         inline token(const token& t);
 
+        //- Move construct. The original token is left as UNDEFINED.
+        inline token(token&& t);
+
         //- Construct punctuation character token
         inline explicit token(punctuationToken p);
 
-        //- Construct word token
-        inline explicit token(const word& w);
-
-        //- Construct string token
-        inline explicit token(const string& str);
-
         //- Construct label token
         inline explicit token(const label val);
 
@@ -304,16 +317,16 @@ public:
         //- Construct doubleScalar token
         inline explicit token(const doubleScalar val);
 
+        //- Construct word token by copying word contents
+        inline explicit token(const word& w);
+
+        //- Construct string token by copying string contents
+        inline explicit token(const string& str);
+
 
         //- Construct punctuation character token
         inline token(punctuationToken p, const label lineNumber);
 
-        //- Construct word token
-        inline token(const word& w, const label lineNumber);
-
-        //- Construct string token
-        inline token(const string& str, const label lineNumber);
-
         //- Construct label token
         inline token(const label val, const label lineNumber);
 
@@ -323,6 +336,12 @@ public:
         //- Construct doubleScalar token
         inline token(const doubleScalar val, const label lineNumber);
 
+        //- Construct word token by copying word contents
+        inline token(const word& w, const label lineNumber);
+
+        //- Construct string token by copying string contents
+        inline token(const string& str, const label lineNumber);
+
         //- Construct from Istream
         token(Istream& is);
 
@@ -331,61 +350,143 @@ public:
     inline ~token();
 
 
+    // Static member functions
+
+        //- True if the character is a punctuation separator (eg, in ISstream).
+        //  Since it could also start a number, SUBTRACT is not included as
+        //  a separator.
+        //
+        //  \param c the character to test, passed as int for consistency with
+        //      isdigit, isspace etc.
+        inline static bool isseparator(int c);
+
+
     // Member functions
 
+    // Status
+
+        //- Return the name of the token type
+        word name() const;
+
+        //- Return the token type
+        inline tokenType type() const;
+
+        //- Change the token type, for similar types.
+        //  This can be used to change between string-like variants
+        //  (eg, STRING, VARIABLE, VERBATIMSTRING)
+        //  To change types entirely (eg, STRING to DOUBLE_SCALAR),
+        //  use the corresponding assignment operator.
+        //
+        //  \return true if the change was successful or no change was required
+        inline bool setType(const tokenType variant);
+
+        //- The line number for the token
+        inline label lineNumber() const;
+
+        //- The line number for the token
+        inline label& lineNumber();
+
+        //- True if token is not UNDEFINED or ERROR
+        inline bool good() const;
+
+        //- True if token is UNDEFINED
+        inline bool undefined() const;
+
+        //- True if token is ERROR
+        inline bool error() const;
+
+        //- True if token is PUNCTUATION
+        inline bool isPunctuation() const;
+
+        //- True if token is PUNCTUATION and isseparator
+        inline bool isSeparator() const;
+
+        //- True if token is LABEL
+        inline bool isLabel() const;
+
+        //- True if token is FLOAT_SCALAR
+        inline bool isFloatScalar() const;
+
+        //- True if token is DOUBLE_SCALAR
+        inline bool isDoubleScalar() const;
+
+        //- True if token is FLOAT_SCALAR or DOUBLE_SCALAR
+        inline bool isScalar() const;
+
+        //- True if token is LABEL, FLOAT_SCALAR or DOUBLE_SCALAR
+        inline bool isNumber() const;
+
+        //- True if token is WORD
+        inline bool isWord() const;
+
+        //- True if token is STRING, VARIABLE or VERBATIMSTRING
+        inline bool isString() const;
+
+        //- True if token is VARIABLE
+        inline bool isVariable() const;
+
+        //- True if token is COMPOUND
+        inline bool isCompound() const;
+
+
     // Access
 
-            inline tokenType type() const;
+        //- Return punctuation character.
-            inline tokenType& type();
+        //  Report FatalIOError and return \b \\0 if token is not PUNCTUATION
-
-            inline bool good() const;
-            inline bool undefined() const;
-            inline bool error() const;
-
-            inline bool isPunctuation() const;
         inline punctuationToken pToken() const;
 
-            inline bool isWord() const;
+        //- Return label value.
-            inline const word& wordToken() const;
+        //  Report FatalIOError and return \b 0 if token is not LABEL
-
-            inline bool isVariable() const;
-
-            inline bool isString() const;
-            inline const string& stringToken() const;
-
-            inline bool isLabel() const;
         inline label labelToken() const;
 
-            inline bool isFloatScalar() const;
+        //- Return float value.
+        //  Report FatalIOError and return \b 0.0 if token is not FLOAT_SCALAR
         inline floatScalar floatScalarToken() const;
 
-            inline bool isDoubleScalar() const;
+        //- Return double value.
+        //  Report FatalIOError and return \b 0.0 if token is not DOUBLE_SCALAR
         inline doubleScalar doubleScalarToken() const;
 
-            inline bool isScalar() const;
+        //- Return float or double value.
+        //  Report FatalIOError and return \b 0.0 if token is not a
+        //  FLOAT_SCALAR or DOUBLE_SCALAR
         inline scalar scalarToken() const;
 
-            inline bool isNumber() const;
+        //- Return label, float or double value.
+        //  Report FatalIOError and return \b 0.0 if token is not a
+        //  LABEL, FLOAT_SCALAR or DOUBLE_SCALAR
         inline scalar number() const;
 
-            inline bool isCompound() const;
+        //- Return const reference to the word contents.
-            inline const compound& compoundToken() const;
+        //  Report FatalIOError and return \b "" if token is not a WORD
-            compound& transferCompoundToken(const Istream& is);
+        inline const word& wordToken() const;
 
-            inline label lineNumber() const;
+        //- Return const reference to the string contents.
-            inline label& lineNumber();
+        //  Report FatalIOError and return \b "" if token is not a
+        //  STRING, VARIABLE or VERBATIMSTRING
+        inline const string& stringToken() const;
+
+        //- Read access for compound token
+        inline const compound& compoundToken() const;
+
+        //- Return reference to compound token and decrease its internal
+        //- refCound accordingly.
+        //  The Istream is used for reference error messages only.
+        compound& transferCompoundToken(const Istream& is);
 
 
     // Edit
 
-            //- Set bad
+        //- Clear token and set to be in an error state.
         inline void setBad();
 
+        //- Swap token contents: type, data, line-number
+        inline void swap(token& tok);
+
 
     // Info
 
-            //- Return info proxy.
+        //- Return info proxy for printing token information to a stream
-            //  Used to print token information to a stream
         InfoProxy<token> info() const
         {
             return *this;
@@ -396,43 +497,69 @@ public:
 
     // Assignment
 
-            inline void operator=(const token& t);
+        //- Copy assign
+        inline void operator=(const token& tok);
 
+        //- Move assign
+        inline void operator=(token&& tok);
+
+        //- Copy assign from punctuation
         inline void operator=(const punctuationToken p);
 
-            inline void operator=(word* wPtr);
+        //- Copy assign from label
-            inline void operator=(const word& w);
-
-            inline void operator=(string* strPtr);
-            inline void operator=(const string& str);
-
         inline void operator=(const label val);
+
+        //- Copy assign from float
         inline void operator=(const floatScalar val);
+
+        //- Copy assign from double
         inline void operator=(const doubleScalar val);
 
-            inline void operator=(compound* compPtr);
+        //- Copy assign from word
+        inline void operator=(const word& w);
+
+        //- Copy assign from string
+        inline void operator=(const string& str);
+
+        //- Move assign from word
+        inline void operator=(word&& w);
+
+        //- Move assign from string
+        inline void operator=(string&& str);
+
+        //- Transfer word pointer to the token
+        //  \deprecated in favour of using move assign from word
+        inline void operator=(word* wordPtr);
+
+        //- Transfer string pointer to the token
+        //  \deprecated in favour of using move assign from string
+        inline void operator=(string* stringPtr);
+
+        //- Assign compound with reference counting to token
+        inline void operator=(compound* compoundPtr);
 
 
     // Equality
 
         inline bool operator==(const token& t) const;
+
         inline bool operator==(const punctuationToken p) const;
-            inline bool operator==(const word& w) const;
-            inline bool operator==(const string& str) const;
         inline bool operator==(const label val) const;
         inline bool operator==(const floatScalar val) const;
         inline bool operator==(const doubleScalar val) const;
+        inline bool operator==(const word& w) const;
+        inline bool operator==(const string& str) const;
 
 
     // Inequality
 
         inline bool operator!=(const token& t) const;
         inline bool operator!=(const punctuationToken p) const;
-            inline bool operator!=(const word& w) const;
-            inline bool operator!=(const string& str) const;
         inline bool operator!=(const label val) const;
         inline bool operator!=(const floatScalar val) const;
         inline bool operator!=(const doubleScalar val) const;
+        inline bool operator!=(const word& w) const;
+        inline bool operator!=(const string& str) const;
 
 
     // IOstream operators

src/OpenFOAM/db/IOstreams/token/tokenI.H

@@ -3,7 +3,7 @@
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
     \\  /    A nd           | Copyright (C) 2011-2016 OpenFOAM Foundation
-     \\/     M anipulation  |
+     \\/     M anipulation  | Copyright (C) 2017 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -23,36 +23,54 @@ License
 
 \*---------------------------------------------------------------------------*/
 
+#include <algorithm>
+
 // * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
 
+inline void Foam::token::setUndefined()
+{
+    type_ = tokenType::UNDEFINED;
+    data_.int64Val = 0; // bit-wise zero for union content
+    // leave lineNumber untouched - may still be needed
+}
+
+
 inline void Foam::token::clear()
 {
-    if (type_ == tokenType::WORD)
+    switch (type_)
     {
-        delete wordTokenPtr_;
+        case tokenType::WORD:
+        {
+            delete data_.wordPtr;
+            break;
         }
-    else if
+
-    (
+        case tokenType::STRING:
-        type_ == tokenType::STRING
+        case tokenType::VARIABLE:
-     || type_ == tokenType::VARIABLE
+        case tokenType::VERBATIMSTRING:
-     || type_ == tokenType::VERBATIMSTRING
-    )
         {
-        delete stringTokenPtr_;
+            delete data_.stringPtr;
+            break;
         }
-    else if (type_ == tokenType::COMPOUND)
+
+        case tokenType::COMPOUND:
         {
-        if (compoundTokenPtr_->unique())
+            if (data_.compoundPtr->unique())
             {
-            delete compoundTokenPtr_;
+                delete data_.compoundPtr;
             }
             else
             {
-            compoundTokenPtr_->refCount::operator--();
+                data_.compoundPtr->refCount::operator--();
             }
+            break;
         }
 
-    type_ = tokenType::UNDEFINED;
+        default:
+            break;
+    }
+
+    setUndefined();
 }
 
 
@@ -60,55 +78,59 @@ inline void Foam::token::clear()
 
 inline Foam::token::token()
 :
+    data_(), // bit-wise zero for union content
     type_(tokenType::UNDEFINED),
     lineNumber_(0)
 {}
 
 
-inline Foam::token::token(const token& t)
+inline Foam::token::token(const token& tok)
 :
-    type_(t.type_),
+    data_(tok.data_), // bit-wise copy of union content
-    lineNumber_(t.lineNumber_)
+    type_(tok.type_),
+    lineNumber_(tok.lineNumber_)
 {
+    // Fundamental: values already handled by bit-wise copy
+    // Pointer: duplicate content or increase refCount
+
     switch (type_)
     {
-        case tokenType::UNDEFINED:
-        break;
-
-        case tokenType::PUNCTUATION:
-            punctuationToken_ = t.punctuationToken_;
-        break;
-
         case tokenType::WORD:
-            wordTokenPtr_ = new word(*t.wordTokenPtr_);
+        {
+            data_.wordPtr = new word(*tok.data_.wordPtr);
             break;
+        }
 
         case tokenType::STRING:
         case tokenType::VARIABLE:
         case tokenType::VERBATIMSTRING:
-            stringTokenPtr_ = new string(*t.stringTokenPtr_);
+        {
-        break;
+            data_.stringPtr = new string(*tok.data_.stringPtr);
-
-        case tokenType::LABEL:
-            labelToken_ = t.labelToken_;
-        break;
-
-        case tokenType::FLOAT_SCALAR:
-            floatScalarToken_ = t.floatScalarToken_;
-        break;
-
-        case tokenType::DOUBLE_SCALAR:
-            doubleScalarToken_ = t.doubleScalarToken_;
-        break;
-
-        case tokenType::COMPOUND:
-            compoundTokenPtr_ = t.compoundTokenPtr_;
-            compoundTokenPtr_->refCount::operator++();
-        break;
-
-        case tokenType::ERROR:
             break;
         }
+
+        case tokenType::COMPOUND:
+        {
+            // Identical pointers, but increase the refCount
+            data_.compoundPtr = tok.data_.compoundPtr;
+            data_.compoundPtr->refCount::operator++();
+            break;
+        }
+
+        default:
+            break;
+    }
+}
+
+
+inline Foam::token::token(token&& tok)
+:
+    data_(tok.data_), // bit-wise copy of union content
+    type_(tok.type_),
+    lineNumber_(tok.lineNumber_)
+{
+    tok.setUndefined(); // zero the union content without any checking
+    tok.lineNumber_ = 0;
 }
 
 
@@ -150,50 +172,62 @@ inline Foam::token::token(const doubleScalar val)
 
 inline Foam::token::token(punctuationToken p, label lineNumber)
 :
+    data_(),
     type_(tokenType::PUNCTUATION),
-    punctuationToken_(p),
     lineNumber_(lineNumber)
-{}
+{
-
+    data_.punctuationVal = p;
-
+}
-inline Foam::token::token(const word& w, label lineNumber)
-:
-    type_(tokenType::WORD),
-    wordTokenPtr_(new word(w)),
-    lineNumber_(lineNumber)
-{}
-
-
-inline Foam::token::token(const string& str, label lineNumber)
-:
-    type_(tokenType::STRING),
-    stringTokenPtr_(new string(str)),
-    lineNumber_(lineNumber)
-{}
 
 
 inline Foam::token::token(const label val, label lineNumber)
 :
+    data_(),
     type_(tokenType::LABEL),
-    labelToken_(val),
     lineNumber_(lineNumber)
-{}
+{
+    data_.labelVal = val;
+}
 
 
 inline Foam::token::token(const floatScalar val, label lineNumber)
 :
+    data_(),
     type_(tokenType::FLOAT_SCALAR),
-    floatScalarToken_(val),
     lineNumber_(lineNumber)
-{}
+{
+    data_.floatVal = val;
+}
 
 
 inline Foam::token::token(const doubleScalar val, label lineNumber)
 :
+    data_(),
     type_(tokenType::DOUBLE_SCALAR),
-    doubleScalarToken_(val),
     lineNumber_(lineNumber)
-{}
+{
+    data_.doubleVal = val;
+}
+
+
+inline Foam::token::token(const word& w, label lineNumber)
+:
+    data_(),
+    type_(tokenType::WORD),
+    lineNumber_(lineNumber)
+{
+    data_.wordPtr = new word(w);
+}
+
+
+inline Foam::token::token(const string& str, label lineNumber)
+:
+    data_(),
+    type_(tokenType::STRING),
+    lineNumber_(lineNumber)
+{
+    data_.stringPtr = new string(str);
+}
 
 
 // * * * * * * * * * * * * * * * * Destructor  * * * * * * * * * * * * * * * //
@@ -206,134 +240,179 @@ inline Foam::token::~token()
 
 // * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
 
+inline void Foam::token::swap(token& tok)
+{
+    std::swap(data_, tok.data_);
+    std::swap(type_, tok.type_);
+    std::swap(lineNumber_, tok.lineNumber_);
+}
+
+
 inline Foam::token::tokenType Foam::token::type() const
 {
     return type_;
 }
 
-inline Foam::token::tokenType&  Foam::token::type()
+
+inline bool Foam::token::setType(token::tokenType variant)
 {
-    return type_;
+    if (type_ == variant)
+    {
+        // No change required
+        return true;
+    }
+
+    switch (variant)
+    {
+        case tokenType::STRING:
+        case tokenType::VARIABLE:
+        case tokenType::VERBATIMSTRING:
+        {
+            switch (type_)
+            {
+                // could also go from WORD to STRING etc - to be decided
+                case tokenType::STRING:
+                case tokenType::VARIABLE:
+                case tokenType::VERBATIMSTRING:
+                    type_ = variant;
+                    return true;
+                    break;
+
+                default:
+                    return false;
+                    break;
+            }
+        }
+
+        default:
+            break;
+    }
+
+    return false;
 }
 
+
+inline Foam::label Foam::token::lineNumber() const
+{
+    return lineNumber_;
+}
+
+
+inline Foam::label& Foam::token::lineNumber()
+{
+    return lineNumber_;
+}
+
+
 inline bool Foam::token::good() const
 {
-    return (type_ != tokenType::ERROR && type_ != tokenType::UNDEFINED);
+    return (type_ != tokenType::UNDEFINED && type_ != tokenType::ERROR);
 }
 
+
 inline bool Foam::token::undefined() const
 {
     return (type_ == tokenType::UNDEFINED);
 }
 
+
 inline bool Foam::token::error() const
 {
     return (type_ == tokenType::ERROR);
 }
 
+
 inline bool Foam::token::isPunctuation() const
 {
     return (type_ == tokenType::PUNCTUATION);
 }
 
+
 inline Foam::token::punctuationToken Foam::token::pToken() const
 {
     if (type_ == tokenType::PUNCTUATION)
     {
-        return punctuationToken_;
+        return data_.punctuationVal;
     }
-    else
+
-    {
     parseError("punctuation character");
     return NULL_TOKEN;
-    }
 }
 
-inline bool Foam::token::isWord() const
-{
-    return (type_ == tokenType::WORD);
-}
 
-inline const Foam::word& Foam::token::wordToken() const
+inline bool Foam::token::isseparator(int c)
 {
-    if (type_ == tokenType::WORD)
+    switch (c)
     {
-        return *wordTokenPtr_;
+        case token::END_STATEMENT :
-    }
+        case token::BEGIN_LIST :
-    else
+        case token::END_LIST :
+        case token::BEGIN_SQR :
+        case token::END_SQR :
+        case token::BEGIN_BLOCK :
+        case token::END_BLOCK :
+        case token::COLON :
+        case token::COMMA :
+        case token::ASSIGN :
+        case token::ADD :
+        // Excluded token::SUBTRACT since it could start a number
+        case token::MULTIPLY :
+        case token::DIVIDE :
         {
-        parseError(word::typeName);
+            return true;
-        return word::null;
         }
+
+        default:
+            break;
+    }
+
+    return false;
 }
 
-inline bool Foam::token::isVariable() const
-{
-    return (type_ == tokenType::VARIABLE);
-}
 
-inline bool Foam::token::isString() const
+inline bool Foam::token::isSeparator() const
 {
     return
     (
-        type_ == tokenType::STRING
+        type_ == tokenType::PUNCTUATION
-     || type_ == tokenType::VARIABLE
+     && isseparator(data_.punctuationVal)
-     || type_ == tokenType::VERBATIMSTRING
     );
 }
 
-inline const Foam::string& Foam::token::stringToken() const
-{
-    if
-    (
-        type_ == tokenType::STRING
-     || type_ == tokenType::VARIABLE
-     || type_ == tokenType::VERBATIMSTRING
-    )
-    {
-        return *stringTokenPtr_;
-    }
-    else
-    {
-        parseError(string::typeName);
-        return string::null;
-    }
-}
 
 inline bool Foam::token::isLabel() const
 {
     return (type_ == tokenType::LABEL);
 }
 
+
 inline Foam::label Foam::token::labelToken() const
 {
     if (type_ == tokenType::LABEL)
     {
-        return labelToken_;
+        return data_.labelVal;
     }
-    else
+
-    {
     parseError(pTraits<label>::typeName);
     return 0;
-    }
 }
 
+
 inline bool Foam::token::isFloatScalar() const
 {
     return (type_ == tokenType::FLOAT_SCALAR);
 }
 
+
 inline Foam::floatScalar Foam::token::floatScalarToken() const
 {
     if (type_ == tokenType::FLOAT_SCALAR)
     {
-        return floatScalarToken_;
+        return data_.floatVal;
     }
-    else
+
-    {
     parseError("floatScalar");
     return 0.0;
-    }
 }
 
 
@@ -342,17 +421,16 @@ inline bool Foam::token::isDoubleScalar() const
     return (type_ == tokenType::DOUBLE_SCALAR);
 }
 
+
 inline Foam::doubleScalar Foam::token::doubleScalarToken() const
 {
     if (type_ == tokenType::DOUBLE_SCALAR)
     {
-        return doubleScalarToken_;
+        return data_.doubleVal;
     }
-    else
+
-    {
     parseError("doubleScalar");
     return 0.0;
-    }
 }
 
 
@@ -365,72 +443,112 @@ inline bool Foam::token::isScalar() const
     );
 }
 
+
 inline Foam::scalar Foam::token::scalarToken() const
 {
     if (type_ == tokenType::FLOAT_SCALAR)
     {
-        return floatScalarToken_;
+        return data_.floatVal;
     }
     else if (type_ == tokenType::DOUBLE_SCALAR)
     {
-        return doubleScalarToken_;
+        return data_.doubleVal;
     }
-    else
+
-    {
     parseError(pTraits<scalar>::typeName);
     return 0.0;
-    }
 }
 
+
 inline bool Foam::token::isNumber() const
 {
     return (type_ == tokenType::LABEL || isScalar());
 }
 
+
 inline Foam::scalar Foam::token::number() const
 {
-    if (type_ == tokenType::LABEL)
+    if (isLabel())
     {
-        return labelToken_;
+        return labelToken();
     }
-    else if (isScalar())
+    if (isScalar())
     {
         return scalarToken();
     }
-    else
+
-    {
     parseError("number (label or scalar)");
     return 0.0;
-    }
 }
 
+
+inline bool Foam::token::isWord() const
+{
+    return (type_ == tokenType::WORD);
+}
+
+
+inline const Foam::word& Foam::token::wordToken() const
+{
+    if (type_ == tokenType::WORD)
+    {
+        return *data_.wordPtr;
+    }
+
+    parseError(word::typeName);
+    return word::null;
+}
+
+
+inline bool Foam::token::isVariable() const
+{
+    return (type_ == tokenType::VARIABLE);
+}
+
+
+inline bool Foam::token::isString() const
+{
+    return
+    (
+        type_ == tokenType::STRING
+     || type_ == tokenType::VARIABLE
+     || type_ == tokenType::VERBATIMSTRING
+    );
+}
+
+
+inline const Foam::string& Foam::token::stringToken() const
+{
+    if
+    (
+        type_ == tokenType::STRING
+     || type_ == tokenType::VARIABLE
+     || type_ == tokenType::VERBATIMSTRING
+    )
+    {
+        return *data_.stringPtr;
+    }
+
+    parseError(string::typeName);
+    return string::null;
+}
+
+
 inline bool Foam::token::isCompound() const
 {
     return (type_ == tokenType::COMPOUND);
 }
 
+
 inline const Foam::token::compound& Foam::token::compoundToken() const
 {
     if (type_ == tokenType::COMPOUND)
     {
-        return *compoundTokenPtr_;
+        return *data_.compoundPtr;
     }
-    else
+
-    {
     parseError("compound");
-        return *compoundTokenPtr_;
+    return *data_.compoundPtr;  // This is questionable.
-    }
-}
-
-
-inline Foam::label Foam::token::lineNumber() const
-{
-    return lineNumber_;
-}
-
-inline Foam::label& Foam::token::lineNumber()
-{
-    return lineNumber_;
 }
 
 
@@ -443,117 +561,148 @@ inline void Foam::token::setBad()
 
 // * * * * * * * * * * * * * * * Member Operators  * * * * * * * * * * * * * //
 
-inline void Foam::token::operator=(const token& t)
+inline void Foam::token::operator=(const token& tok)
 {
     clear();
-    type_ = t.type_;
+
+    type_ = tok.type_;
+    data_ = tok.data_;  // bit-wise copy of union content
+    lineNumber_ = tok.lineNumber_;
+
+    // Fundamental: values already handled by bit-wise copy
+    // Pointer: duplicate content or increase refCount
 
     switch (type_)
     {
-        case tokenType::UNDEFINED:
-        break;
-
-        case tokenType::PUNCTUATION:
-            punctuationToken_ = t.punctuationToken_;
-        break;
-
         case tokenType::WORD:
-            wordTokenPtr_ = new word(*t.wordTokenPtr_);
+        {
+            data_.wordPtr = new word(*tok.data_.wordPtr);
+        }
         break;
 
         case tokenType::STRING:
         case tokenType::VARIABLE:
         case tokenType::VERBATIMSTRING:
-            stringTokenPtr_ = new string(*t.stringTokenPtr_);
+        {
-        break;
+            data_.stringPtr = new string(*tok.data_.stringPtr);
-
+        }
-        case tokenType::LABEL:
-            labelToken_ = t.labelToken_;
-        break;
-
-        case tokenType::FLOAT_SCALAR:
-            floatScalarToken_ = t.floatScalarToken_;
-        break;
-
-        case tokenType::DOUBLE_SCALAR:
-            doubleScalarToken_ = t.doubleScalarToken_;
         break;
 
         case tokenType::COMPOUND:
-            compoundTokenPtr_ = t.compoundTokenPtr_;
+        {
-            compoundTokenPtr_->refCount::operator++();
+            // Identical pointers, but increase the refCount
+            data_.compoundPtr = tok.data_.compoundPtr;
+            data_.compoundPtr->refCount::operator++();
+        }
         break;
 
-        case tokenType::ERROR:
+        default:
             break;
     }
-
-    lineNumber_ = t.lineNumber_;
 }
 
+
+inline void Foam::token::operator=(token&& tok)
+{
+    clear();
+    lineNumber_ = 0;
+    swap(tok);
+}
+
+
 inline void Foam::token::operator=(const punctuationToken p)
 {
     clear();
     type_ = tokenType::PUNCTUATION;
-    punctuationToken_ = p;
+    data_.punctuationVal = p;
 }
 
-inline void Foam::token::operator=(word* wPtr)
-{
-    clear();
-    type_ = tokenType::WORD;
-    wordTokenPtr_ = wPtr;
-}
-
-inline void Foam::token::operator=(const word& w)
-{
-    operator=(new word(w));
-}
-
-inline void Foam::token::operator=(string* strPtr)
-{
-    clear();
-    type_ = tokenType::STRING;
-    stringTokenPtr_ = strPtr;
-}
-
-inline void Foam::token::operator=(const string& str)
-{
-    operator=(new string(str));
-}
 
 inline void Foam::token::operator=(const label val)
 {
     clear();
     type_ = tokenType::LABEL;
-    labelToken_ = val;
+    data_.labelVal = val;
 }
 
+
 inline void Foam::token::operator=(const floatScalar val)
 {
     clear();
     type_ = tokenType::FLOAT_SCALAR;
-    floatScalarToken_ = val;
+    data_.floatVal = val;
 }
 
+
 inline void Foam::token::operator=(const doubleScalar val)
 {
     clear();
     type_ = tokenType::DOUBLE_SCALAR;
-    doubleScalarToken_ = val;
+    data_.doubleVal = val;
 }
 
-inline void Foam::token::operator=(Foam::token::compound* compPtr)
+
+inline void Foam::token::operator=(word* wordPtr)
+{
+    clear();
+    type_ = tokenType::WORD;
+    data_.wordPtr = wordPtr;
+}
+
+
+inline void Foam::token::operator=(string* stringPtr)
+{
+    clear();
+    type_ = tokenType::STRING;
+    data_.stringPtr = stringPtr;
+}
+
+
+inline void Foam::token::operator=(const word& w)
+{
+    clear();
+    type_ = tokenType::WORD;
+    data_.wordPtr = new word(w);
+}
+
+
+inline void Foam::token::operator=(const string& str)
+{
+    clear();
+    type_ = tokenType::STRING;
+    data_.stringPtr = new string(str);
+}
+
+
+inline void Foam::token::operator=(word&& w)
+{
+    clear();
+    type_ = tokenType::WORD;
+    data_.wordPtr = new word;
+    data_.wordPtr->swap(w);
+}
+
+
+inline void Foam::token::operator=(string&& s)
+{
+    clear();
+    type_ = tokenType::STRING;
+    data_.stringPtr = new string;
+    data_.stringPtr->swap(s);
+}
+
+
+inline void Foam::token::operator=(Foam::token::compound* compoundPtr)
 {
     clear();
     type_ = tokenType::COMPOUND;
-    compoundTokenPtr_ = compPtr;
+    data_.compoundPtr = compoundPtr;
 }
 
 
-inline bool Foam::token::operator==(const token& t) const
+inline bool Foam::token::operator==(const token& tok) const
 {
-    if (type_ != t.type_)
+    if (type_ != tok.type_)
     {
         return false;
     }
@@ -564,27 +713,27 @@ inline bool Foam::token::operator==(const token& t) const
             return true;
 
         case tokenType::PUNCTUATION:
-            return punctuationToken_ == t.punctuationToken_;
+            return data_.punctuationVal == tok.data_.punctuationVal;
+
+        case tokenType::LABEL:
+            return data_.labelVal == tok.data_.labelVal;
+
+        case tokenType::FLOAT_SCALAR:
+            return equal(data_.floatVal, tok.data_.floatVal);
+
+        case tokenType::DOUBLE_SCALAR:
+            return equal(data_.doubleVal, tok.data_.doubleVal);
 
         case tokenType::WORD:
-            return *wordTokenPtr_ == *t.wordTokenPtr_;
+            return *data_.wordPtr == *tok.data_.wordPtr;
 
         case tokenType::STRING:
         case tokenType::VARIABLE:
         case tokenType::VERBATIMSTRING:
-            return *stringTokenPtr_ == *t.stringTokenPtr_;
+            return *data_.stringPtr == *tok.data_.stringPtr;
-
-        case tokenType::LABEL:
-            return labelToken_ == t.labelToken_;
-
-        case tokenType::FLOAT_SCALAR:
-            return equal(floatScalarToken_, t.floatScalarToken_);
-
-        case tokenType::DOUBLE_SCALAR:
-            return equal(doubleScalarToken_, t.doubleScalarToken_);
 
         case tokenType::COMPOUND:
-            return compoundTokenPtr_ == t.compoundTokenPtr_;
+            return data_.compoundPtr == tok.data_.compoundPtr;
 
         case tokenType::ERROR:
             return true;
@@ -593,16 +742,19 @@ inline bool Foam::token::operator==(const token& t) const
     return false;
 }
 
+
 inline bool Foam::token::operator==(const punctuationToken p) const
 {
-    return (type_ == tokenType::PUNCTUATION && punctuationToken_ == p);
+    return (type_ == tokenType::PUNCTUATION && data_.punctuationVal == p);
 }
 
+
 inline bool Foam::token::operator==(const word& w) const
 {
     return (type_ == tokenType::WORD && wordToken() == w);
 }
 
+
 inline bool Foam::token::operator==(const string& str) const
 {
     return
@@ -616,67 +768,77 @@ inline bool Foam::token::operator==(const string& str) const
     );
 }
 
+
 inline bool Foam::token::operator==(const label val) const
 {
     return
     (
         type_ == tokenType::LABEL
-     && labelToken_ == val
+     && data_.labelVal == val
     );
 }
 
+
 inline bool Foam::token::operator==(const floatScalar val) const
 {
     return
     (
         type_ == tokenType::FLOAT_SCALAR
-     && equal(floatScalarToken_, val)
+     && equal(data_.floatVal, val)
     );
 }
 
+
 inline bool Foam::token::operator==(const doubleScalar val) const
 {
     return
     (
         type_ == tokenType::DOUBLE_SCALAR
-     && equal(doubleScalarToken_, val)
+     && equal(data_.doubleVal, val)
     );
 }
 
+
 inline bool Foam::token::operator!=(const token& t) const
 {
     return !operator==(t);
 }
 
+
 inline bool Foam::token::operator!=(const punctuationToken p) const
 {
     return !operator==(p);
 }
 
-inline bool Foam::token::operator!=(const word& w) const
-{
-    return !operator==(w);
-}
-
-inline bool Foam::token::operator!=(const string& str) const
-{
-    return !operator==(str);
-}
 
 inline bool Foam::token::operator!=(const label val) const
 {
     return !operator==(val);
 }
 
+
 inline bool Foam::token::operator!=(const floatScalar val) const
 {
     return !operator==(val);
 }
 
+
 inline bool Foam::token::operator!=(const doubleScalar val) const
 {
     return !operator==(val);
 }
 
 
+inline bool Foam::token::operator!=(const word& w) const
+{
+    return !operator==(w);
+}
+
+
+inline bool Foam::token::operator!=(const string& str) const
+{
+    return !operator==(str);
+}
+
+
 // ************************************************************************* //

src/OpenFOAM/db/IOstreams/token/tokenIO.C

@@ -3,7 +3,7 @@
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
     \\  /    A nd           | Copyright (C) 2011-2015 OpenFOAM Foundation
-     \\/     M anipulation  |
+     \\/     M anipulation  | Copyright (C) 2017 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -25,32 +25,129 @@ License
 
 #include "error.H"
 #include "token.H"
-
-#include "IOstreams.H"
 #include "scalar.H"
+#include "IOstreams.H"
+
+// * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+template<class OS>
+static OS& printTokenInfo(OS& os, const token& tok)
+{
+    os  << "on line " << tok.lineNumber() << ": ";
+
+    switch (tok.type())
+    {
+        case token::tokenType::UNDEFINED:
+            os  << "undefined token";
+        break;
+
+        case token::tokenType::PUNCTUATION:
+            os  << "punctuation '" << tok.pToken() << '\'';
+        break;
+
+        case token::tokenType::LABEL:
+            os  << "label " << tok.labelToken();
+        break;
+
+        case token::tokenType::FLOAT_SCALAR:
+            os  << "float " << tok.floatScalarToken();
+        break;
+
+        case token::tokenType::DOUBLE_SCALAR:
+            os  << "double " << tok.doubleScalarToken();
+        break;
+
+        case token::tokenType::WORD:
+            os  << "word '" << tok.wordToken() << '\'';
+        break;
+
+        case token::tokenType::STRING:
+            os  << "string " << tok.stringToken();
+        break;
+
+        case token::tokenType::VARIABLE:
+            os  << "variable " << tok.stringToken();
+        break;
+
+        case token::tokenType::VERBATIMSTRING:
+            os  << "verbatim string " << tok.stringToken();
+        break;
+
+        case token::tokenType::COMPOUND:
+        {
+            if (tok.compoundToken().empty())
+            {
+                os  << "empty ";
+            }
+            os  << "compound of type "
+                << tok.compoundToken().type();
+        }
+        break;
+
+        case token::tokenType::ERROR:
+            os  << "error";
+        break;
+
+        default:
+            os  << "unknown token type '" << int(tok.type()) << '\'';
+            break;
+    }
+
+    return os;
+}
+} // End namespace Foam
+
 
 // * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
 
 Foam::token::token(Istream& is)
 :
-    type_(tokenType::UNDEFINED)
+    token()
 {
     is.read(*this);
 }
 
 
-// * * * * * * * * * * * * * * * IOstream Operators  * * * * * * * * * * * * //
+// * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
 
-Foam::Istream& Foam::operator>>(Istream& is, token& t)
+Foam::word Foam::token::name() const
 {
-    t.clear();
+    switch (type_)
-    return is.read(t);
+    {
+        case token::tokenType::UNDEFINED: return "undefined";
+        case token::tokenType::PUNCTUATION: return "punctuation";
+        case token::tokenType::LABEL: return "label";
+        case token::tokenType::FLOAT_SCALAR: return "float";
+        case token::tokenType::DOUBLE_SCALAR: return "double";
+        case token::tokenType::WORD: return "word";
+        case token::tokenType::STRING: return "string";
+        case token::tokenType::VERBATIMSTRING: return "verbatim";
+        case token::tokenType::VARIABLE: return "variable";
+        case token::tokenType::COMPOUND: return "compound";
+        case token::tokenType::ERROR: return "error";
+
+        default:
+            break;
+    }
+
+    return "unknown(" + std::to_string(int(type_)) + ")";
 }
 
 
-Foam::Ostream& Foam::operator<<(Ostream& os, const token& t)
+// * * * * * * * * * * * * * * * IOstream Operators  * * * * * * * * * * * * //
+
+Foam::Istream& Foam::operator>>(Istream& is, token& tok)
 {
-    switch (t.type_)
+    tok.clear();
+    return is.read(tok);
+}
+
+
+Foam::Ostream& Foam::operator<<(Ostream& os, const token& tok)
+{
+    switch (tok.type_)
     {
         case token::tokenType::UNDEFINED:
             os << "UNDEFINED";
@@ -59,37 +156,37 @@ Foam::Ostream& Foam::operator<<(Ostream& os, const token& t)
         break;
 
         case token::tokenType::PUNCTUATION:
-            os << t.punctuationToken_;
+            os << tok.data_.punctuationVal;
+        break;
+
+        case token::tokenType::LABEL:
+            os << tok.data_.labelVal;
+        break;
+
+        case token::tokenType::FLOAT_SCALAR:
+            os << tok.data_.floatVal;
+        break;
+
+        case token::tokenType::DOUBLE_SCALAR:
+            os << tok.data_.doubleVal;
         break;
 
         case token::tokenType::WORD:
-            os << *t.wordTokenPtr_;
+            os << *tok.data_.wordPtr;
         break;
 
         case token::tokenType::STRING:
         case token::tokenType::VERBATIMSTRING:
-            os << *t.stringTokenPtr_;
+            os << *tok.data_.stringPtr;
         break;
 
         case token::tokenType::VARIABLE:
             // Behaviour differs according to stream type
-            os.write(t);
+            os.write(tok);
-        break;
-
-        case token::tokenType::LABEL:
-            os << t.labelToken_;
-        break;
-
-        case token::tokenType::FLOAT_SCALAR:
-            os << t.floatScalarToken_;
-        break;
-
-        case token::tokenType::DOUBLE_SCALAR:
-            os << t.doubleScalarToken_;
         break;
 
         case token::tokenType::COMPOUND:
-            os << *t.compoundTokenPtr_;
+            os << *tok.data_.compoundPtr;
         break;
 
         case token::tokenType::ERROR:
@@ -135,144 +232,14 @@ Foam::Ostream& Foam::operator<<(Ostream& os, const token::compound& ct)
 
 ostream& Foam::operator<<(ostream& os, const InfoProxy<token>& ip)
 {
-    const token& t = ip.t_;
+    return printTokenInfo(os, ip.t_);
-
-    os  << "on line " << t.lineNumber();
-
-    switch (t.type())
-    {
-        case token::tokenType::UNDEFINED:
-            os  << " an undefined token";
-        break;
-
-        case token::tokenType::PUNCTUATION:
-            os  << " the punctuation token " << '\'' << t.pToken() << '\'';
-        break;
-
-        case token::tokenType::WORD:
-            os  << " the word " << '\'' << t.wordToken() << '\'';
-        break;
-
-        case token::tokenType::STRING:
-            os  << " the string " << t.stringToken();
-        break;
-
-        case token::tokenType::VARIABLE:
-            os  << " the variable " << t.stringToken();
-        break;
-
-        case token::tokenType::VERBATIMSTRING:
-            os  << " the verbatim string " << t.stringToken();
-        break;
-
-        case token::tokenType::LABEL:
-            os  << " the label " << t.labelToken();
-        break;
-
-        case token::tokenType::FLOAT_SCALAR:
-            os  << " the floatScalar " << t.floatScalarToken();
-        break;
-
-        case token::tokenType::DOUBLE_SCALAR:
-            os  << " the doubleScalar " << t.doubleScalarToken();
-        break;
-
-        case token::tokenType::COMPOUND:
-        {
-            if (t.compoundToken().empty())
-            {
-                os  << " the empty compound of type "
-                    << t.compoundToken().type();
-            }
-            else
-            {
-                os  << " the compound of type "
-                    << t.compoundToken().type();
-            }
-        }
-        break;
-
-        case token::tokenType::ERROR:
-            os  << " an error";
-        break;
-
-        default:
-            os  << " an unknown token type " << '\'' << int(t.type()) << '\'';
-    }
-
-    return os;
 }
 
 
 template<>
 Foam::Ostream& Foam::operator<<(Ostream& os, const InfoProxy<token>& ip)
 {
-    const token& t = ip.t_;
+    return printTokenInfo(os, ip.t_);
-
-    os  << "on line " << t.lineNumber();
-
-    switch (t.type())
-    {
-        case token::tokenType::UNDEFINED:
-            os  << " an undefined token";
-        break;
-
-        case token::tokenType::PUNCTUATION:
-            os  << " the punctuation token " << '\'' << t.pToken() << '\'';
-        break;
-
-        case token::tokenType::WORD:
-            os  << " the word " << '\'' << t.wordToken() << '\'';
-        break;
-
-        case token::tokenType::STRING:
-            os  << " the string " << t.stringToken();
-        break;
-
-        case token::tokenType::VARIABLE:
-            os  << " the variable " << t.stringToken();
-        break;
-
-        case token::tokenType::VERBATIMSTRING:
-            os  << " the verbatim string " << t.stringToken();
-        break;
-
-        case token::tokenType::LABEL:
-            os  << " the label " << t.labelToken();
-        break;
-
-        case token::tokenType::FLOAT_SCALAR:
-            os  << " the floatScalar " << t.floatScalarToken();
-        break;
-
-        case token::tokenType::DOUBLE_SCALAR:
-            os  << " the doubleScalar " << t.doubleScalarToken();
-        break;
-
-        case token::tokenType::COMPOUND:
-        {
-            if (t.compoundToken().empty())
-            {
-                os  << " the empty compound of type "
-                    << t.compoundToken().type();
-            }
-            else
-            {
-                os  << " the compound of type "
-                    << t.compoundToken().type();
-            }
-        }
-        break;
-
-        case token::tokenType::ERROR:
-            os  << " an error";
-        break;
-
-        default:
-            os  << " an unknown token type "  << '\'' << int(t.type()) << '\'';
-    }
-
-    return os;
 }
 
 

src/OpenFOAM/memory/refCount/refCount.H

@@ -36,8 +36,6 @@ See also
 #ifndef refCount_H
 #define refCount_H
 
-#include "bool.H"
-
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
 namespace Foam
@@ -50,20 +48,10 @@ namespace Foam
 class refCount
 {
     // Private data
-
     int count_;
 
 
-    // Private Member Functions
+public:
-
-        //- Dissallow copy
-        refCount(const refCount&);
-
-        //- Dissallow bitwise assignment
-        void operator=(const refCount&);
-
-
-protected:
 
     // Constructors
 
@@ -74,8 +62,6 @@ protected:
         {}
 
 
-public:
-
     // Member Functions
 
         //- Return the current reference count
@@ -87,7 +73,7 @@ public:
         //- Return true if the reference count is zero
         bool unique() const
         {
-            return count_ == 0;
+            return !count_;
         }
 
 
@@ -96,25 +82,25 @@ public:
         //- Increment the reference count
         void operator++()
         {
-            count_++;
+            ++count_;
         }
 
         //- Increment the reference count
         void operator++(int)
         {
-            count_++;
+            ++count_;
         }
 
         //- Decrement the reference count
         void operator--()
         {
-            count_--;
+            --count_;
         }
 
         //- Decrement the reference count
         void operator--(int)
         {
-            count_--;
+            --count_;
         }
 };
 

src/OpenFOAM/memory/tmp/tmp.H

@@ -161,7 +161,7 @@ public:
         //- Assignment to pointer changing this tmp to a temporary T
         inline void operator=(T* tPtr);
 
-        //- Assignment transfering the temporary T to this tmp
+        //- Assignment transferring the temporary T to this tmp
         inline void operator=(const tmp<T>& t);
 };