ENH: add token::read(Istream&) method

- can be used to simplify some logic. For example,

      if
      (
          (tok.read(is) && tok.isWord("FoamFile"))
       && (tok.read(is) && tok.isPunctuation(token::BEGIN_BLOCK))
      )
      ...
  vs
      if
      (
          (is.good() && (is >> tok) && tok.isWord("FoamFile")) ...
       && (is.good() && (is >> tok) && tok.isPunctuation(token::BEGIN_BLOCK))
      ) ...

applications/test/ITstream/Test-ITstream.cxx

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2017-2023 OpenCFD Ltd.
+    Copyright (C) 2017-2025 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -74,15 +74,11 @@ Ostream& toString(Ostream& os, const List<char>& list)
 void printTokens(Istream& is)
 {
     label count = 0;
-    token t;
-    while (is.good())
+
+    Info<< "stream tokens:" << endl;
+    for (token tok; tok.read(is); ++count)
     {
-        is >> t;
-        if (t.good())
-        {
-            ++count;
-            Info<< "token: " << t << endl;
-        }
+        Info<< "  : " << tok << endl;
     }
 
     Info<< count << " tokens" << endl;
@@ -455,6 +451,12 @@ int main(int argc, char *argv[])
             "( const char input \"string\" to tokenize )\n"
             "List<label> 5(0 1 2 3 4);";
 
+        // printTokens
+        {
+            ISpanStream is(charInput);
+            printTokens(is);
+        }
+
         string stringInput("( string     ;    input \"string\" to tokenize )");
 
         List<char> listInput

src/OpenFOAM/db/IOstreams/token/token.H

@@ -807,6 +807,10 @@ public:
         //- Swap token contents: type, data, line-number
         inline void swap(token& tok) noexcept;
 
+        //- Read a token from Istream, calls reset() first.
+        //  \returns token state as good() or not.
+        bool read(Istream& is);
+
 
     // Info
 
@@ -922,6 +926,7 @@ public:
 
 // IOstream Operators
 
+//- Read token from Istream. Same as token::read() but returns stream state
 Istream& operator>>(Istream& is, token& tok);
 Ostream& operator<<(Ostream& os, const token::punctuationToken& pt);
 ostream& operator<<(ostream& os, const token::punctuationToken& pt);

src/OpenFOAM/db/IOstreams/token/tokenIO.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2015 OpenFOAM Foundation
-    Copyright (C) 2017-2023 OpenCFD Ltd.
+    Copyright (C) 2017-2025 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -144,6 +144,19 @@ Foam::token::token(Istream& is)
 }
 
 
+// * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
+
+bool Foam::token::read(Istream& is)
+{
+    reset();
+    if (is.good())
+    {
+        is.read(*this);
+    }
+    return good();
+}
+
+
 // * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * //
 
 Foam::word Foam::token::name(const token::tokenType tokType)

src/OpenFOAM/expressions/value/exprValue.C

@@ -50,62 +50,53 @@ namespace Foam
 {
 
 #if 0
-// General way to add tokens for VectorSpace types
+// General way to add tokens for fundamental and VectorSpace types
 // (caller excludes none/invalid)
 template<class Type>
 static void addTokens(tokenList& toks, const Type& val)
 {
-    const direction nCmpt = pTraits<Type>::nComponents;
-    const direction nParen = 2*(pTraits<Type>::rank || (nCmpt > 1) ? 1 : 0);
-
-    const label nOld = toks.size();
-    toks.resize(nOld + label(nCmpt + nParen));
-
-    auto iter = toks.begin(nOld);
-
-    if (nParen)
+    if constexpr (std::is_same_v<bool, Type>)
     {
-        *iter = token::BEGIN_LIST;
-        ++iter;
+        // For bool
+        toks.emplace_back() = token::boolean(val);
     }
-
-    for (direction cmpt = 0; cmpt < nCmpt; ++cmpt)
+    else if constexpr (std::is_arithmetic_v<Type>)
     {
-        *iter = component(val, cmpt);
-        ++iter;
+        // For label, float, double, ...
+        toks.emplace_back() = val;
     }
-
-    if (nParen)
+    else
     {
-        *iter = token::END_LIST;
-        ++iter;
+        constexpr direction nCmpt = pTraits_nComponents<Type>::value;
+
+        constexpr direction nParen =
+            2*(is_vectorspace_v<Type> || (nCmpt > 1) ? 1 : 0);
+
+        const label nOld = toks.size();
+        toks.resize(nOld + label(nCmpt + nParen));
+
+        auto iter = toks.begin(nOld);
+
+        if (nParen)
+        {
+            *iter = token::BEGIN_LIST;
+            ++iter;
+        }
+
+        for (direction cmpt = 0; cmpt < nCmpt; ++cmpt)
+        {
+            *iter = component(val, cmpt);
+            ++iter;
+        }
+
+        if (nParen)
+        {
+            *iter = token::END_LIST;
+            ++iter;
+        }
     }
 }
 
-
-//- Specialized for bool
-template<>
-void addTokens<bool>(tokenList& toks, const bool& val)
-{
-    toks.emplace_back() = token::boolean(val);
-}
-
-
-//- Specialized for label
-template<>
-void addTokens<label>(tokenList& toks, const label& val)
-{
-    toks.emplace_back() = val;
-}
-
-
-//- Specialized for scalar
-template<>
-void addTokens<scalar>(tokenList& toks, const scalar& val)
-{
-    toks.emplace_back() = val;
-}
-
 #endif