ENH: improve stream handling of expansions (#2095)

* removed internal upper limit on word/string length for parsed input.

  - Although it has not caused many problems, no reason to retain
    these limits.
  - simplify some of the internal logic for reading string-like items.
  - localize parsers for better separation from the header

  - expose new function seekCommentEnd_Cstyle(), as useful
    handler of C-style comments

* exclude imbalanced closing ')' from word/variable

  - previously included this into the word/variable, but makes more
    sense to leave on the parser for the following token.

    Prevents content like 'vector (10 20 $zmax);' from being parsed
    as '$zmax)' instead of as '$zmax' followed by a ')'.
    No conceivable reason that the former would actually be desirable,
    but can still be obtained with brace notation: Eg, '${zmax)}'

* consistent handling of ${{ ... }} expressions

  - within a dictionary content, the following construct was
    incorrectly processed:

        value ${{2*sqrt(0.5)}};

    Complains about no dictionary/env variable "{2*sqrt(0.5)}"

    Now trap expressions directly and assign their own token type
    while reading. Later expansion can then be properly passed to
    the exprDriver (evalEntry) instead of incorrectly trying
    variable expansion.

    Does not alter the use of expressions embedded within other
    expansions. Eg, "file${{10*2}}"

* improve #eval { ... } brace slurping

  - the initial implementation of this was rudimentary and simply
    grabbed everything until the next '}'.  Now continue to grab
    content until braces are properly balanced

    Eg, the content:   value #eval{${radius}*2};

    would have previously terminated prematurely with "${radius" for
    the expression!

NOTE:
    both the ${{ expr }} parsed input and the #eval { ... } input
    discard C/C++ comments during reading to reduce intermediate
    overhead for content that will be discarded before evaluation
    anyhow.

* tighten recognition of verbatim strings and expressions.

  - parser was previously sloppy and would have accepted content such
    as "# { ..." (for example) as an verbatim string introducer.
    Now only accept parse if there are no intermediate characters
    discarded.

src/OpenFOAM/db/IOstreams/Sstreams/ISstream.C

@@ -30,6 +30,7 @@ License
 #include "int.H"
 #include "token.H"
 #include <cctype>
+#include <cstring>
 
 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
 
@@ -42,14 +43,14 @@ namespace
 {
 
 // Convert a single character to a word with length 1
-inline static Foam::word charToWord(char c)
+inline Foam::word charToWord(char c)
 {
     return Foam::word(std::string(1, c), false);
 }
 
 
 // Permit slash-scoping of entries
-static inline bool validVariableChar(char c)
+inline bool validVariableChar(char c)
 {
     return (Foam::word::valid(c) || c == '/');
 }
@@ -59,23 +60,51 @@ static inline bool validVariableChar(char c)
 
 // * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * * //
 
+bool Foam::ISstream::seekCommentEnd_Cstyle()
+{
+    // Search for end of C-style comment - "*/"
+
+    // Can use getLine(nullptr, '*') in the logic,
+    // but written out looks less obscure
+
+    char c = 0;
+    bool star = false;
+
+    while (get(c))
+    {
+        if (c == '*')
+        {
+            star = true;
+        }
+        else if (star)
+        {
+            star = false;
+            if (c == '/')
+            {
+                // Matched "*/"
+                return true;
+            }
+        }
+    }
+
+    // Exhausted stream without finding "*/" sequence
+    return false;
+}
+
+
 char Foam::ISstream::nextValid()
 {
     char c = 0;
 
-    while (true)
+    // Get next non-whitespace character
+    while (get(c))
     {
-        // Get next non-whitespace character
+        if (isspace(c))
-        while (get(c) && isspace(c))
-        {}
-
-        // Return if stream is bad - ie, previous get() failed
-        if (bad() || isspace(c))
         {
-            return 0;
+            continue;
         }
 
-        // Is this the start of a C/C++ comment?
+        // Check if this starts a C/C++ comment
         if (c == '/')
         {
             if (!get(c))
@@ -86,37 +115,15 @@ char Foam::ISstream::nextValid()
 
             if (c == '/')
             {
-                // C++ style single-line comment - skip through past end-of-line
+                // C++ comment: discard through newline
-                while (get(c) && c != '\n')
+                (void) getLine(nullptr, '\n');
-                {}
             }
             else if (c == '*')
             {
-                // Within a C-style comment
+                // C-style comment: discard through to "*/" ending
-                while (true)
+                if (!seekCommentEnd_Cstyle())
                 {
-                    // Search for end of C-style comment - '*/'
+                    return 0;
-                    if (get(c) && c == '*')
-                    {
-                        if (get(c))
-                        {
-                            if (c == '/')
-                            {
-                                // matched '*/'
-                                break;
-                            }
-                            else if (c == '*')
-                            {
-                                // check again
-                                putback(c);
-                            }
-                        }
-                    }
-
-                    if (!good())
-                    {
-                        return 0;
-                    }
                 }
             }
             else
@@ -137,28 +144,261 @@ char Foam::ISstream::nextValid()
 }
 
 
-void Foam::ISstream::readWordToken(token& t)
+// * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
+
+namespace Foam
 {
-    word val;
+
-    if (read(val).bad())
+// Read a verbatim string (excluding block delimiters),
+// continuing until a closing "#}" has been found.
+//
+// The leading "#{" removed from stream prior to calling.
+static ISstream& readVerbatim
+(
+    ISstream& is,
+    std::string& str
+)
+{
+    constexpr const unsigned bufLen = 8000;
+    static char buf[bufLen];
+
+    unsigned nChar = 0;
+    char c;
+
+    str.clear();
+    while (is.get(c))
     {
-        t.setBad();
+        if (c == token::HASH)
-    }
+        {
-    else if (token::compound::isCompound(val))
+            char nextC;
-    {
+            is.get(nextC);
-        t = token::compound::New(val, *this).ptr();
+            if (nextC == token::END_BLOCK)
-    }
+            {
-    else
+                // Found closing "#}" sequence
-    {
+                str.append(buf, nChar);
-        t = std::move(val); // Move contents to token
+                return is;
+            }
+            else
+            {
+                // Re-analyze the character
+                is.putback(nextC);
+            }
+        }
+
+        buf[nChar++] = c;
+        if (nChar == bufLen)  // Flush full buffer
+        {
+            str.append(buf, nChar);
+            nChar = 0;
+        }
     }
+
+
+    // Abnormal exit of the loop
+    str.append(buf, nChar);  // Finalize pending content
+    strncpy(buf, str.c_str(), errLen);
+    buf[errLen] = '\0';
+
+    FatalIOErrorInFunction(is)
+        << "Problem while reading verbatim \"" << buf
+        << "...\" [after " << str.length() << " chars]\n"
+        << exit(FatalIOError);
+
+    return is;
 }
 
 
+// Read a variable or expression.
+// Handles "$var" and "${var}" forms, permits '/' scoping character.
+// Also handles "${{expr}}".
+//
+// Return the token type or ERROR
+//
+// The leading "${" or "$c" removed from stream prior to calling.
+static token::tokenType readVariable
+(
+    ISstream& is,
+    std::string& str,
+    char c  // Next character after '$'
+)
+{
+    constexpr const unsigned bufLen = 1024;
+    static char buf[bufLen];
+
+    token::tokenType tokType(token::tokenType::VARIABLE);
+
+    // The first two characters are known:
+    buf[0] = token::DOLLAR;
+    buf[1] = c;
+
+    unsigned nChar = 2; // Starts with two characters
+    unsigned depth = 0; // Depth of {..} nesting
+
+    str.clear();
+    if (c == token::BEGIN_BLOCK)
+    {
+        // Processing '${variable}' or '${{expr}}'
+        ++depth;
+
+        int lookahead = is.peek();
+        if (lookahead == token::BEGIN_BLOCK)
+        {
+            // Looks like '${{expr...'
+            tokType = token::tokenType::EXPRESSION;
+        }
+        else if (lookahead == token::END_BLOCK)
+        {
+            // Looks like '${}'
+            IOWarningInFunction(is)
+                << "Ignoring empty ${}" << endl;
+            return token::tokenType::ERROR;
+        }
+
+        while (is.get(c))
+        {
+            buf[nChar++] = c;
+
+            if (c == token::BEGIN_BLOCK)
+            {
+                ++depth;
+            }
+            else if (c == token::END_BLOCK)
+            {
+                --depth;
+                if (!depth)
+                {
+                    // Found closing '}' character
+                    str.append(buf, nChar);
+                    return tokType;
+                }
+            }
+            else if (c == '/' && tokType == token::tokenType::EXPRESSION)
+            {
+                // Strip C/C++ comments from expressions
+                // Note: could also peek instead of get/putback
+
+                if (!is.get(c))
+                {
+                    break;  // Premature end of stream
+                }
+                else if (c == '/')
+                {
+                    --nChar;  // Remove initial '/' from buffer
+
+                    // C++ comment: discard through newline
+                    (void) is.getLine(nullptr, '\n');
+                }
+                else if (c == '*')
+                {
+                    --nChar;  // Remove initial '/' from buffer
+
+                    // C-style comment: seek "*/" ending
+                    if (!is.seekCommentEnd_Cstyle())
+                    {
+                        break;  // Premature end of stream
+                    }
+                }
+                else
+                {
+                    // Re-analyze the character
+                    is.putback(c);
+                }
+            }
+
+            if (nChar == bufLen)  // Flush full buffer
+            {
+                str.append(buf, nChar);
+                nChar = 0;
+            }
+        }
+
+
+        // Abnormal exit of the loop
+
+        str.append(buf, nChar);  // Finalize pending content
+        strncpy(buf, str.c_str(), errLen);
+        buf[errLen] = '\0';
+
+        FatalIOErrorInFunction(is)
+            << "stream terminated while reading variable '" << buf
+            << "...' [after " << str.length() << " chars]\n"
+            << exit(FatalIOError);
+
+        return token::tokenType::ERROR;
+    }
+    else if (validVariableChar(c))
+    {
+        // Processing '$variable'
+
+        while (is.get(c))
+        {
+            if (!validVariableChar(c))
+            {
+                is.putback(c);
+                break;
+            }
+
+            if (c == token::BEGIN_LIST)
+            {
+                ++depth;
+            }
+            else if (c == token::END_LIST)
+            {
+                if (!depth)
+                {
+                    // Closed ')' without opening '(':
+                    // - don't consider it part of our input
+                    is.putback(c);
+                    break;
+                }
+                --depth;
+            }
+
+            buf[nChar++] = c;
+            if (nChar == bufLen)  // Flush full buffer
+            {
+                str.append(buf, nChar);
+                nChar = 0;
+            }
+        }
+
+        str.append(buf, nChar);  // Finalize pending content
+
+        if (depth)
+        {
+            strncpy(buf, str.c_str(), errLen);
+            buf[errLen] = '\0';
+
+            IOWarningInFunction(is)
+                << "Missing " << depth
+                << " closing ')' while parsing" << nl << nl
+                << buf << endl;
+        }
+
+        return tokType;
+    }
+    else
+    {
+        // Invalid character. Terminate string (for message)
+
+        buf[nChar--] = '\0';
+
+        IOWarningInFunction(is)
+            << "Ignoring bad variable name: " << buf << nl << endl;
+    }
+
+    return token::tokenType::ERROR;
+}
+
+} // End namespace Foam
+
+
+// * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * * //
+
 Foam::Istream& Foam::ISstream::read(token& t)
 {
-    constexpr const unsigned maxLen = 128; // Max length for labels/scalars
+    constexpr const unsigned bufLen = 128; // Max length for labels/scalars
-    static char buf[maxLen];
+    static char buf[bufLen];
 
     // Return the put back token if it exists
     if (Istream::getBack(t))
@@ -209,7 +449,7 @@ Foam::Istream& Foam::ISstream::read(token& t)
         }
 
         // String: enclosed by double quotes.
-        case token::BEGIN_STRING :
+        case token::DQUOTE :
         {
             putback(c);
 
@@ -226,21 +466,21 @@ Foam::Istream& Foam::ISstream::read(token& t)
             return *this;
         }
 
-        // Possible verbatim string or dictionary functionEntry
+        // Verbatim string '#{ .. #}' or dictionary '#directive'
         case token::HASH :
         {
             char nextC;
-            if (read(nextC).bad())
+            int lookahead = peek();
-            {
+
-                // Return lone '#' as word
+            if (lookahead == token::BEGIN_BLOCK)
-                t = charToWord(c);
-            }
-            else if (nextC == token::BEGIN_BLOCK)
             {
                 // Verbatim string: #{ ... #}
+                // Token stored without the surrounding delimiters
+
+                (void) get(nextC);  // Discard '{' lookahead
 
                 string val;
-                if (readVerbatim(val).bad())
+                if (readVerbatim(*this, val).bad())
                 {
                     t.setBad();
                 }
@@ -250,9 +490,14 @@ Foam::Istream& Foam::ISstream::read(token& t)
                     t.setType(token::tokenType::VERBATIM);
                 }
             }
-            else
+            else if (read(nextC).bad())
             {
-                // Word beginning with '#'. Eg, "#include"
+                // Return lone '#' as word
+                t = charToWord(c);
+            }
+            else if (word::valid(nextC))
+            {
+                // Directive (wordToken) beginning with '#'. Eg, "#include"
                 // Put back both so that '#...' is included in the directive
 
                 putback(nextC);
@@ -269,34 +514,44 @@ Foam::Istream& Foam::ISstream::read(token& t)
                     t.setType(token::tokenType::DIRECTIVE);
                 }
             }
+            else
+            {
+                // '#' followed by non-word. Just ignore leading '#'?
+                putback(nextC);
+
+                IOWarningInFunction(*this)
+                    << "Invalid sequence #" << char(nextC)
+                    << " ... ignoring the leading '#'" << nl << endl;
+            }
 
             return *this;
         }
 
-        // Dictionary variable (as rvalue)
+        // Dictionary variable or ${{ expression }}
         case token::DOLLAR :
         {
             char nextC;
             if (read(nextC).bad())
             {
-                // Return lone '$' as word
+                // Return lone '$' as word. Could also ignore
                 t = charToWord(c);
             }
             else
             {
-                // Put back both so that '$...' is included in the variable
+                // NB: the parser is slightly generous here.
-                putback(nextC);
+                // It will also accept '$  {' as input.
-                putback(c);
+                // - to be revisited (2021-05-17)
 
                 string val;
-                if (readVariable(val).bad())
+                token::tokenType tokType = readVariable(*this, val, nextC);
+                if (tokType == token::tokenType::ERROR)
                 {
                     t.setBad();
                 }
                 else
                 {
                     t = std::move(val); // Move contents to token
-                    t.setType(token::tokenType::VARIABLE);
+                    t.setType(tokType);
                 }
             }
 
@@ -340,14 +595,14 @@ Foam::Istream& Foam::ISstream::read(token& t)
                 }
 
                 buf[nChar++] = c;
-                if (nChar == maxLen)
+                if (nChar == bufLen)
                 {
                     // Runaway argument - avoid buffer overflow
-                    buf[maxLen-1] = '\0';
+                    buf[bufLen-1] = '\0';
 
                     FatalIOErrorInFunction(*this)
-                        << "number '" << buf << "...'\n"
+                        << "Number '" << buf << "...'\n"
-                        << "    is too long (max. " << maxLen << " characters)"
+                        << "    is too long (max. " << bufLen << " characters)"
                         << exit(FatalIOError);
 
                     t.setBad();
@@ -397,7 +652,20 @@ Foam::Istream& Foam::ISstream::read(token& t)
         default:
         {
             putback(c);
-            readWordToken(t);
+
+            word val;
+            if (read(val).bad())
+            {
+                t.setBad();
+            }
+            else if (token::compound::isCompound(val))
+            {
+                t = token::compound::New(val, *this).ptr();
+            }
+            else
+            {
+                t = std::move(val); // Move contents to token
+            }
 
             return *this;
         }
@@ -414,20 +682,22 @@ Foam::Istream& Foam::ISstream::read(char& c)
 
 Foam::Istream& Foam::ISstream::read(word& str)
 {
-    constexpr const unsigned maxLen = 1024;
+    constexpr const unsigned bufLen = 1024;
-    static char buf[maxLen];
+    static char buf[bufLen];
 
     unsigned nChar = 0;
-    unsigned depth = 0;  // Track depth of (..) nesting
+    unsigned depth = 0;  // Depth of (..) nesting
     char c;
 
-    while
+    str.clear();
-    (
+    while (get(c))
-        (nChar < maxLen)
-     && get(c)
-     && word::valid(c)
-    )
     {
+        if (!word::valid(c))
+        {
+            putback(c);
+            break;
+        }
+
         if (c == token::BEGIN_LIST)
         {
             ++depth;
@@ -436,42 +706,40 @@ Foam::Istream& Foam::ISstream::read(word& str)
         {
             if (!depth)
             {
-                break;  // Closed ')' without an opening '(' ? ... stop
+                // Closed ')' without opening '(':
+                // - don't consider it part of our input
+                putback(c);
+                break;
             }
             --depth;
         }
 
         buf[nChar++] = c;
+        if (nChar == bufLen)  // Flush full buffer
+        {
+            str.append(buf, nChar);
+            nChar = 0;
+        }
     }
 
-    if (nChar >= maxLen)
+    str.append(buf, nChar);  // Finalize pending content
-    {
-        buf[errLen] = '\0';
-
-        FatalIOErrorInFunction(*this)
-            << "word '" << buf << "...'\n"
-            << "    is too long (max. " << maxLen << " characters)"
-            << exit(FatalIOError);
-
-        return *this;
-    }
-
-    buf[nChar] = '\0';  // Terminate string
 
     if (bad())
     {
         // Could probably skip this check
+
+        strncpy(buf, str.c_str(), errLen);
         buf[errLen] = '\0';
 
         FatalIOErrorInFunction(*this)
-            << "Problem while reading word '" << buf << "...' after "
+            << "Problem while reading word '" << buf
-            << nChar << " characters\n"
+            << "...' [after " << str.length() << " chars]\n"
             << exit(FatalIOError);
 
         return *this;
     }
 
-    if (nChar == 0)
+    if (str.empty())
     {
         FatalIOErrorInFunction(*this)
             << "Invalid first character found : " << c
@@ -479,25 +747,25 @@ Foam::Istream& Foam::ISstream::read(word& str)
     }
     else if (depth)
     {
+        strncpy(buf, str.c_str(), errLen);
+        buf[errLen] = '\0';
+
         IOWarningInFunction(*this)
             << "Missing " << depth
             << " closing ')' while parsing" << nl << nl
             << buf << nl << endl;
     }
 
-    // Finalize: content already validated, assign without additional checks.
-    str.assign(buf, nChar);
-    putback(c);
-
     return *this;
 }
 
 
 Foam::Istream& Foam::ISstream::read(string& str)
 {
-    constexpr const unsigned maxLen = 1024;
+    constexpr const unsigned bufLen = 1024;
-    static char buf[maxLen];
+    static char buf[bufLen];
 
+    unsigned nChar = 0;
     char c;
 
     if (!get(c))
@@ -510,7 +778,7 @@ Foam::Istream& Foam::ISstream::read(string& str)
     }
 
     // Note, we could also handle single-quoted strings here (if desired)
-    if (c != token::BEGIN_STRING)
+    if (c != token::DQUOTE)
     {
         FatalIOErrorInFunction(*this)
             << "Incorrect start of string character found : " << c
@@ -519,26 +787,25 @@ Foam::Istream& Foam::ISstream::read(string& str)
         return *this;
     }
 
-    unsigned nChar = 0;
+    str.clear();
     bool escaped = false;
-
+    while (get(c))
-    while
-    (
-        (nChar < maxLen)
-     && get(c)
-    )
     {
-        if (c == token::END_STRING)
+        if (c == '\\')
+        {
+            escaped = !escaped;  // Toggle state (retains backslashes)
+        }
+        else if (c == token::DQUOTE)
         {
             if (escaped)
             {
                 escaped = false;
-                --nChar;    // Overwrite backslash
+                --nChar;  // Overwrite backslash
             }
             else
             {
                 // Done reading
-                str.assign(buf, nChar);
+                str.append(buf, nChar);
                 return *this;
             }
         }
@@ -547,253 +814,44 @@ Foam::Istream& Foam::ISstream::read(string& str)
             if (escaped)
             {
                 escaped = false;
-                --nChar;    // Overwrite backslash
+                --nChar;  // Overwrite backslash
             }
             else
             {
-                buf[errLen] = buf[nChar] = '\0';
+                str.append(buf, nChar);  // Finalize pending content
+                strncpy(buf, str.c_str(), errLen);
+                buf[errLen] = '\0';
 
                 FatalIOErrorInFunction(*this)
-                    << "found '\\n' while reading string \""
+                    << "Unescaped '\\n' while reading string \"" << buf
-                    << buf << "...\""
+                    << "...\" [after " << str.length() << " chars]\n"
                     << exit(FatalIOError);
 
                 return *this;
             }
         }
-        else if (c == '\\')
-        {
-            escaped = !escaped;    // toggle state (retains backslashes)
-        }
         else
         {
             escaped = false;
         }
 
         buf[nChar++] = c;
+        if (nChar == bufLen)  // Flush full buffer
+        {
+            // Keep lookback character (eg, for backslash escaping)
+            str.append(buf, nChar-1);
+            nChar = 1;
+            buf[0] = c;
+        }
     }
 
-    if (nChar >= maxLen)
-    {
-        buf[errLen] = '\0';
-
-        FatalIOErrorInFunction(*this)
-            << "string \"" << buf << "...\"\n"
-            << "    is too long (max. " << maxLen << " characters)"
-            << exit(FatalIOError);
-
-        return *this;
-    }
 
+    // Abnormal exit of the loop
     // Don't worry about a dangling backslash if string terminated prematurely
-    buf[errLen] = buf[nChar] = '\0';
 
-    FatalIOErrorInFunction(*this)
+    str.append(buf, nChar);  // Finalize pending content
-        << "Problem while reading string \"" << buf << "...\""
+    strncpy(buf, str.c_str(), errLen);
-        << exit(FatalIOError);
+    buf[errLen] = '\0';
-
-    return *this;
-}
-
-
-Foam::Istream& Foam::ISstream::readVariable(std::string& str)
-{
-    constexpr const unsigned maxLen = 1024;
-    static char buf[maxLen];
-
-    unsigned nChar = 0;
-    unsigned depth = 0; // Track depth of (..) or {..} nesting
-    char c;
-
-    // First character must be '$'
-    if (!get(c) || c != token::DOLLAR)
-    {
-        FatalIOErrorInFunction(*this)
-            << "Invalid first character found : " << c << nl
-            << exit(FatalIOError);
-    }
-    buf[nChar++] = c;
-
-    // Next character should also exist.
-    // This should never fail, since it was checked before calling.
-    if (!get(c))
-    {
-        str.assign(buf, nChar);
-
-        IOWarningInFunction(*this)
-            << "Truncated variable name : " << str << nl;
-
-        return *this;
-    }
-    buf[nChar++] = c;
-
-    str.clear();
-    if (c == token::BEGIN_BLOCK)
-    {
-        // Processing ${...} style.
-        ++depth;
-
-        // Could check that the next char is good and not one of '{}'
-        // since this would indicate "${}", "${{..." or truncated "${"
-
-        while (get(c))
-        {
-            buf[nChar++] = c;
-            if (nChar == maxLen)
-            {
-                str.append(buf, nChar);
-                nChar = 0;
-            }
-            if (c == token::BEGIN_BLOCK)
-            {
-                ++depth;
-            }
-            else if (c == token::END_BLOCK)
-            {
-                --depth;
-                if (!depth)
-                {
-                    // Found closing '}' character
-                    str.append(buf, nChar);
-                    return *this;
-                }
-            }
-        }
-
-        // Should never reach here on normal input
-
-        str.append(buf, nChar);  // Finalize pending buffer input
-
-        nChar = str.length();
-        if (str.length() > errLen)
-        {
-            str.erase(errLen);
-        }
-
-        FatalIOErrorInFunction(*this)
-            << "stream terminated while reading variable '"
-            << str.c_str() << "...' [" << nChar << "]\n"
-            << exit(FatalIOError);
-
-        return *this;
-    }
-    else if (validVariableChar(c))
-    {
-        // Processing $var style
-
-        while
-        (
-            (nChar < maxLen) && get(c)
-         && (validVariableChar(c))
-        )
-        {
-            if (c == token::BEGIN_LIST)
-            {
-                ++depth;
-            }
-            else if (c == token::END_LIST)
-            {
-                if (!depth)
-                {
-                    break;  // Closed ')' without an opening '(' ? ... stop
-                }
-                --depth;
-            }
-
-            buf[nChar++] = c;
-        }
-    }
-    else
-    {
-        // Invalid character. Terminate string (for message) without
-        // including the invalid character in the count.
-
-        buf[nChar--] = '\0';
-
-        IOWarningInFunction(*this)
-            << "Bad variable name: " << buf << nl << endl;
-    }
-
-    if (nChar >= maxLen)
-    {
-        buf[errLen] = '\0';
-
-        FatalIOErrorInFunction(*this)
-            << "variable '" << buf << "...'\n"
-            << "    is too long (max. " << maxLen << " characters)"
-            << exit(FatalIOError);
-
-        return *this;
-    }
-
-    buf[nChar] = '\0';  // Terminate string
-
-    if (bad())
-    {
-        // Could probably skip this check
-        buf[errLen] = '\0';
-
-        FatalIOErrorInFunction(*this)
-            << "Problem while reading variable '" << buf << "...' after "
-            << nChar << " characters\n"
-            << exit(FatalIOError);
-
-        return *this;
-    }
-
-    if (depth)
-    {
-        IOWarningInFunction(*this)
-            << "Missing " << depth
-            << " closing ')' while parsing" << nl << nl
-            << buf << nl << endl;
-    }
-
-    // Finalize
-    str.assign(buf, nChar);
-    putback(c);
-
-    return *this;
-}
-
-
-Foam::Istream& Foam::ISstream::readVerbatim(std::string& str)
-{
-    constexpr const unsigned maxLen = 8000;
-    static char buf[maxLen];
-
-    unsigned nChar = 0;
-    char c;
-
-    str.clear();
-    while (get(c))
-    {
-        if (c == token::HASH)
-        {
-            char nextC;
-            get(nextC);
-            if (nextC == token::END_BLOCK)
-            {
-                // Found closing "#}" sequence
-                str.append(buf, nChar);
-                return *this;
-            }
-            else
-            {
-                putback(nextC);
-            }
-        }
-
-        buf[nChar++] = c;
-        if (nChar == maxLen)
-        {
-            str.append(buf, nChar);
-            nChar = 0;
-        }
-    }
-
-    // Truncated terminated prematurely
-    buf[errLen] = buf[nChar] = '\0';
 
     FatalIOErrorInFunction(*this)
         << "Problem while reading string \"" << buf << "...\""

src/OpenFOAM/db/IOstreams/Sstreams/ISstream.H

@@ -69,18 +69,6 @@ class ISstream
         //- Get the next valid character
         char nextValid();
 
-        //- Read a word token
-        void readWordToken(token& t);
-
-        //- Read a verbatim string (excluding block delimiters).
-        //  The leading "#{" has been removed prior to calling,
-        //  continues until the closing "#}" has been found.
-        Istream& readVerbatim(std::string& str);
-
-        //- Read a variable name starting with '$'.
-        //  Handles "$var" and "${var}" forms, permits '/' scoping character.
-        Istream& readVariable(std::string& str);
-
         //- No copy assignment
         void operator=(const ISstream&) = delete;
 
@@ -137,6 +125,13 @@ public:
         virtual ios_base::fmtflags flags() const;
 
 
+    // Special-purpose Functions
+
+        //- Discard until end of C-style comment '*/'
+        //  \return False if stream exhausted before finding the comment end
+        bool seekCommentEnd_Cstyle();
+
+
         // Read Functions
 
             //- Raw, low-level get character function.

src/OpenFOAM/db/IOstreams/Sstreams/OSstream.C

@@ -133,7 +133,7 @@ Foam::Ostream& Foam::OSstream::writeQuoted
 
 
     // Output with surrounding quotes and backslash escaping
-    os_ << token::BEGIN_STRING;
+    os_ << token::DQUOTE;
 
     unsigned backslash = 0;
     for (auto iter = str.cbegin(); iter != str.cend(); ++iter)
@@ -150,7 +150,7 @@ Foam::Ostream& Foam::OSstream::writeQuoted
             ++lineNumber_;
             ++backslash;    // backslash escape for newline
         }
-        else if (c == token::END_STRING)
+        else if (c == token::DQUOTE)
         {
             ++backslash;    // backslash escape for quote
         }
@@ -167,7 +167,7 @@ Foam::Ostream& Foam::OSstream::writeQuoted
 
     // silently drop any trailing backslashes
     // they would otherwise appear like an escaped end-quote
-    os_ << token::END_STRING;
+    os_ << token::DQUOTE;
 
     setState(os_.rdstate());
     return *this;

src/OpenFOAM/db/dictionary/functionEntries/evalEntry/evalEntry.C

@@ -56,60 +56,163 @@ namespace functionEntries
 } // End namespace Foam
 
 
-// * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
+// * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
+
+namespace
+{
+    // This is akin to a SafeIOWarning, which does not yet exist
+    inline void safeIOWarning
+    (
+        const Foam::IOstream& is,
+        const std::string& msg
+    )
+    {
+        std::cerr
+            << "--> FOAM Warning :\n"
+            << "    Reading \"" << is.name() << "\" at line "
+            << is.lineNumber() << '\n'
+            << "    " << msg << std::endl;
+    }
+
+} // End anonymous namespace
+
+
+namespace Foam
+{
+
+// Slurp a string until a closing '}' is found.
+// Track balanced bracket/brace pairs, with max stack depth of 60.
+static bool slurpUntilBalancedBrace(ISstream& is, std::string& str)
+{
+    constexpr const unsigned bufLen = 1024;
+    static char buf[bufLen];
+
+    is.fatalCheck(FUNCTION_NAME);
+
+    unsigned nChar = 0;
+    unsigned depth = 1; // Initial '{' already seen by caller
+    char c;
+
+    str.clear();
+    while (is.get(c))
+    {
+        buf[nChar++] = c;
+
+        if (c == token::BEGIN_BLOCK)
+        {
+            ++depth;
+        }
+        else if (c == token::END_BLOCK)
+        {
+            --depth;
+            if (!depth)
+            {
+                // Closing '}' character - do not include in output
+                --nChar;
+                str.append(buf, nChar);
+                return true;
+            }
+        }
+        else if (c == '/')
+        {
+            // Strip C/C++ comments from expressions
+            // Note: could also peek instead of get/putback
+
+            if (!is.get(c))
+            {
+                break;  // Premature end of stream
+            }
+            else if (c == '/')
+            {
+                --nChar;  // Remove initial '/' from buffer
+
+                // C++ comment: discard through newline
+                (void) is.getLine(nullptr, '\n');
+            }
+            else if (c == '*')
+            {
+                --nChar;  // Remove initial '/' from buffer
+
+                // C-style comment: discard through to "*/" ending
+                if (!is.seekCommentEnd_Cstyle())
+                {
+                    break;  // Premature end of stream
+                }
+            }
+            else
+            {
+                // Reanalyze the char
+                is.putback(c);
+            }
+        }
+
+        if (nChar == bufLen)
+        {
+            str.append(buf, nChar);  // Flush full buffer
+            nChar = 0;
+        }
+    }
+
+
+    // Abnormal exit of the loop
+
+    str.append(buf, nChar);  // Finalize pending content
+
+    safeIOWarning(is, "Premature end while reading expression - missing '}'?");
+
+    is.fatalCheck(FUNCTION_NAME);
+    return false;
+}
+
+} // End namespace Foam
+
+
+// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * //
 
 Foam::tokenList Foam::functionEntries::evalEntry::evaluate
 (
     const dictionary& parentDict,
-    Istream& is
+    const string& inputExpr,
+    label fieldWidth,
+    const Istream& is
 )
 {
-    #ifdef FULLDEBUG
+    // Field width for the result
-    DetailInfo
+    if (fieldWidth < 1)
-        << "Using #eval - line "
-        << is.lineNumber() << " in file " <<  parentDict.name() << nl;
-    #endif
-
-    token tok(is);
-    label fieldWidth(1);  // Field width for the result
-    if (tok.isLabel())
-    {
-        // - #eval INT "expr"
-        // - #eval INT { expr }
-        // - #eval INT #{ expr #}
-        fieldWidth = max(1, tok.labelToken());
-        is >> tok;
-    }
-
-    string s;  // String to evaluate
-    if (tok.isString())
-    {
-        // - #eval "expr"
-        // - #eval #{ expr #}
-        s = tok.stringToken();
-    }
-    else if (tok.isPunctuation(token::BEGIN_BLOCK))
-    {
-        // - #eval { expr }
-        dynamic_cast<ISstream&>(is).getLine(s, token::END_BLOCK);
-    }
-    else
     {
         FatalIOErrorInFunction(is)
-            << "Invalid input for #eval."
+            << "Invalid field width: " << fieldWidth << nl << endl
-               " Expecting a string or block to evaluate, but found" << nl
-            << tok.info() << endl
             << exit(FatalIOError);
     }
 
     #ifdef FULLDEBUG
     DetailInfo
-        << "input: " << s << endl;
+        << "input: " << inputExpr << endl;
     #endif
 
     // Expand with env=true, empty=true, subDict=false
     // with comments stripped.
     // Special handling of $[...] syntax enabled.
+
+    string s;
+
+    // Passed '${{ expr }}' by accident, or on purpuse
+    if
+    (
+        inputExpr[0] == token::DOLLAR
+     && inputExpr[1] == token::BEGIN_BLOCK
+     && inputExpr[2] == token::BEGIN_BLOCK
+     && inputExpr[inputExpr.length()-1] == token::END_BLOCK
+     && inputExpr[inputExpr.length()-2] == token::END_BLOCK
+    )
+    {
+        s.assign(inputExpr, 3, inputExpr.length()-5);
+    }
+    else
+    {
+        s.assign(inputExpr);
+    }
+
     expressions::exprString::inplaceExpand(s, parentDict, true);
     stringOps::inplaceTrim(s);
 
@@ -184,6 +287,60 @@ Foam::tokenList Foam::functionEntries::evalEntry::evaluate
 }
 
 
+Foam::tokenList Foam::functionEntries::evalEntry::evaluate
+(
+    const dictionary& parentDict,
+    Istream& is
+)
+{
+    #ifdef FULLDEBUG
+    DetailInfo
+        << "Using #eval - line "
+        << is.lineNumber() << " in file " <<  parentDict.name() << nl;
+    #endif
+
+    token tok(is);
+    label fieldWidth(1);  // Field width for the result
+    if (tok.isLabel())
+    {
+        // - #eval INT "expr"
+        // - #eval INT { expr }
+        // - #eval INT #{ expr #}
+        fieldWidth = max(1, tok.labelToken());
+        is >> tok;
+    }
+
+    string str;  // The string to evaluate
+    if (tok.isString())
+    {
+        // - #eval "expr"
+        // - #eval #{ expr #}
+        // - #eval ${{ expr }} - wierd but handled
+        str = tok.stringToken();
+    }
+    else if (tok.isPunctuation(token::BEGIN_BLOCK))
+    {
+        // - #eval { expr }
+        slurpUntilBalancedBrace(dynamic_cast<ISstream&>(is), str);
+    }
+    else
+    {
+        FatalIOErrorInFunction(is)
+            << "Invalid input for #eval."
+               " Expecting a string or block to evaluate, but found" << nl
+            << tok.info() << endl
+            << exit(FatalIOError);
+    }
+
+    tokenList toks
+    (
+        evalEntry::evaluate(parentDict, str, fieldWidth, is)
+    );
+
+    return toks;
+}
+
+
 // * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
 
 bool Foam::functionEntries::evalEntry::execute
@@ -201,4 +358,21 @@ bool Foam::functionEntries::evalEntry::execute
 }
 
 
+bool Foam::functionEntries::evalEntry::execute
+(
+    const dictionary& parentDict,
+    primitiveEntry& entry,
+    const string& inputExpr,
+    label fieldWidth,
+    Istream& is
+)
+{
+    tokenList toks(evaluate(parentDict, inputExpr, fieldWidth, is));
+
+    entry.append(std::move(toks), true);  // Lazy resizing
+
+    return true;
+}
+
+
 // ************************************************************************* //

src/OpenFOAM/db/dictionary/functionEntries/evalEntry/evalEntry.H

@@ -87,18 +87,36 @@ class evalEntry
 {
 
     //- Evaluate and return a token list
-    static tokenList evaluate(const dictionary& parentDict, Istream& is);
+    static tokenList evaluate
+    (
+        const dictionary& parentDict,
+        const string& inputExpr,        //!< String to expand and evaluate
+        label fieldWidth,               //!< Field width for the result
+        const Istream& is               //!< For reporting errors
+    );
 
+    //- Evaluate and return a token list
+    static tokenList evaluate(const dictionary& parentDict, Istream& is);
 
 public:
 
-    //- Execute in a primitiveEntry context
+    //- Execute in a primitiveEntry context, extracts token or line
     static bool execute
     (
         const dictionary& parentDict,
         primitiveEntry& thisEntry,
         Istream& is
     );
+
+    //- Execute in a primitiveEntry context, evaluating the given content
+    static bool execute
+    (
+        const dictionary& parentDict,
+        primitiveEntry& entry,
+        const string& inputExpr,
+        label fieldWidth,
+        Istream& is
+    );
 };
 
 

src/OpenFOAM/db/dictionary/primitiveEntry/primitiveEntry.C

@@ -35,7 +35,7 @@ License
 
 //  Find the type/position of the ":-" or ":+" alternative values
 //  Returns 0, '-', '+' corresponding to not-found or ':-' or ':+'
-static inline int findParameterAlternative
+static inline char findParameterAlternative
 (
     const std::string& s,
     std::string::size_type& pos,
@@ -50,7 +50,7 @@ static inline int findParameterAlternative
             if (pos < endPos)
             {
                 // in-range: check for '+' or '-' following the ':'
-                const int altType = s[pos+1];
+                const char altType = s[pos+1];
                 if (altType == '+' || altType == '-')
                 {
                     return altType;
@@ -78,11 +78,13 @@ bool Foam::primitiveEntry::expandVariable
     const dictionary& dict
 )
 {
-    int altType = 0; // Type ('-' or '+') for ":-" or ":+" alternatives
+    char altType = 0; // Type ('-' or '+') for ":-" or ":+" alternatives
     word expanded;
     string altValue;
 
-    if (varName.size() > 1 && varName[0] == token::BEGIN_BLOCK)
+    // Any ${{ expr }} entries have been trapped and processed elsewhere
+
+    if (varName[0] == token::BEGIN_BLOCK && varName.size() > 1)
     {
         // Replace content between {} with string expansion and
         // handle ${parameter:-word} or ${parameter:+word}

src/OpenFOAM/db/dictionary/primitiveEntry/primitiveEntryIO.C

@@ -28,6 +28,7 @@ License
 
 #include "primitiveEntry.H"
 #include "functionEntry.H"
+#include "evalEntry.H"
 
 // * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
 
@@ -63,25 +64,51 @@ bool Foam::primitiveEntry::acceptToken
 
     if (tok.isDirective())
     {
-        // Directive: wordToken starts with '#'
+        // Directive (wordToken) begins with '#'. Eg, "#include"
+        // Remove leading '#' sigil before dispatching
+
         const word& key = tok.wordToken();
 
+        // Min-size is 2: sigil '#' with any content
         accept =
         (
-            disableFunctionEntries
+            (disableFunctionEntries || key.size() < 2)
-         || key.size() == 1
          || !expandFunction(key.substr(1), dict, is)
         );
     }
+    else if (tok.isExpression())
+    {
+        // Expression (stringToken): ${{ expr }}
+        // Surrounding delimiters are stripped as required in evalEntry
+
+        const string& key = tok.stringToken();
+
+        // Min-size is 6: decorators '${{}}' with any content
+        accept =
+        (
+            (disableFunctionEntries || key.size() < 6)
+         || !functionEntries::evalEntry::execute
+            (
+                dict,
+                *this,
+                key,
+                1,      // Field width is 1
+                is      // For error messages
+            )
+        );
+    }
     else if (tok.isVariable())
     {
-        // Variable: stringToken starts with '$'
+        // Variable (stringToken): starts with '$'
+        // Eg, "$varName" or "${varName}"
+        // Remove leading '$' sigil before dispatching
+
         const string& key = tok.stringToken();
 
+        // Min-size is 2: sigil '$' with any content
         accept =
         (
-            disableFunctionEntries
+            (disableFunctionEntries || key.size() < 2)
-         || key.size() == 1
          || !expandVariable(key.substr(1), dict)
         );
     }
@@ -116,7 +143,7 @@ bool Foam::primitiveEntry::read(const dictionary& dict, Istream& is)
     // - similarly, the bitmask is tested *after* decreasing depth
 
     uint64_t balanced = 0u;
-    label depth = 0;
+    int depth = 0;
     token tok;
 
     while
@@ -274,19 +301,18 @@ void Foam::primitiveEntry::write(Ostream& os, const bool contentsOnly) const
         os.writeKeyword(keyword());
     }
 
-    bool addSpace = false;  // Separate from previous tokens with a space
+    bool addSpace = false;  // Separate from previous token with a space
     for (const token& tok : *this)
     {
         if (addSpace) os << token::SPACE;
+        addSpace = true;
 
-        // Try to output token directly, with special handling in Ostreams.
+        // Output token with direct handling in Ostream(s),
-
+        // or use normal '<<' output operator
         if (!os.write(tok))
         {
-            os  << tok;   // Revert to normal '<<' output operator
+            os  << tok;
         }
-
-        addSpace = true;  // Separate from following tokens
     }
 
     if (!contentsOnly)

src/fileFormats/obj/OBJstream.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2012-2016 OpenFOAM Foundation
-    Copyright (C) 2020 OpenCFD Ltd.
+    Copyright (C) 2020-2021 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -120,8 +120,8 @@ Foam::Ostream& Foam::OBJstream::writeQuoted
         return *this;
     }
 
-
+    // Output with surrounding quotes and backslash escaping
-    OFstream::write(static_cast<char>(token::BEGIN_STRING));
+    OFstream::write(static_cast<char>(token::DQUOTE));
 
     unsigned backslash = 0;
     for (auto iter = str.cbegin(); iter != str.cend(); ++iter)
@@ -138,7 +138,7 @@ Foam::Ostream& Foam::OBJstream::writeQuoted
             ++lineNumber_;
             ++backslash;    // backslash escape for newline
         }
-        else if (c == token::END_STRING)
+        else if (c == token::DQUOTE)
         {
             ++backslash;    // backslash escape for quote
         }
@@ -155,7 +155,7 @@ Foam::Ostream& Foam::OBJstream::writeQuoted
 
     // silently drop any trailing backslashes
     // they would otherwise appear like an escaped end-quote
-    OFstream::write(static_cast<char>(token::END_STRING));
+    OFstream::write(static_cast<char>(token::DQUOTE));
 
     return *this;
 }