version tokenize scalar that stops reading on invalid input

applications/test/tokenizeTest/Make/files

@@ -0,0 +1,3 @@
+tokenizeTest.C
+
+EXE = $(FOAM_USER_APPBIN)/tokenizeTest

applications/test/tokenizeTest/Make/options

@@ -0,0 +1,7 @@
+/*
+  check for consistent behaviour with non-optimized code
+  */
+
+EXE_INC = \
+    -DFULLDEBUG -g -O0
+

applications/test/tokenizeTest/tokenizeTest.C

@@ -0,0 +1,87 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | Copyright (C) 1991-2009 OpenCFD Ltd.
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software; you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by the
+    Free Software Foundation; either version 2 of the License, or (at your
+    option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM; if not, write to the Free Software Foundation,
+    Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+
+Application
+
+Description
+    Test the tokenizing of various things
+\*---------------------------------------------------------------------------*/
+
+#include "argList.H"
+#include "IOobject.H"
+#include "IOstreams.H"
+#include "IFstream.H"
+#include "IStringStream.H"
+
+using namespace Foam;
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+//  Main program:
+
+int main(int argc, char *argv[])
+{
+    argList::noParallel();
+    argList::validArgs.insert("string .. stringN");
+    argList::validOptions.insert("file", "name");
+
+    argList args(argc, argv, false, true);
+
+    forAll(args.additionalArgs(), argI)
+    {
+        const string& rawArg = args.additionalArgs()[argI];
+        Info<< "input string: " << rawArg << nl;
+
+        IStringStream is(rawArg);
+        
+        while (is.good())
+        {
+            token tok(is);
+            Info<< "token: " << tok.info() << endl;
+        }
+        
+        Info<< nl;
+        IOobject::writeDivider(Info);
+    }  
+    
+    
+    if (args.optionFound("file"))
+    {
+        IFstream is(args.option("file"));
+        
+        Info<< "tokenizing file: " << args.option("file") << nl;
+
+        while (is.good())
+        {
+            token tok(is);
+            Info<< "token: " << tok.info() << endl;
+        }
+        
+        Info<< nl;
+        IOobject::writeDivider(Info);
+    }
+
+    return 0;
+}
+
+// ************************************************************************* //

src/OpenFOAM/db/IOstreams/Sstreams/ISstream.C

@@ -176,7 +176,7 @@ Foam::Istream& Foam::ISstream::read(token& t)
         //
         // ideally match the equivalent of this regular expression
         //
-        //    /^[-+]?([0-9]+\.?[0-9]*|\.[0-9]+)([Ee][-+]?[0-9]+)?$/
+        //    /[-+]?([0-9]+\.?[0-9]*|\.[0-9]+)([Ee][-+]?[0-9]+)?/
         //
         case '-' :
         case '.' :
@@ -186,69 +186,68 @@ Foam::Istream& Foam::ISstream::read(token& t)
             // has a digit
             bool hasDigit = isdigit(c);
 
-            // has contents that cannot be label
+            // 0 = before a decimal
-            bool notLabel = (c == '.');
+            // 1 = at/after a decimal
-
+            // 2 = in exponent
-            // has contents that cannot be scalar
+            int floatState = 0;
-            bool notScalar = false;
+            if (c == '.')
+            {
+                floatState = 1;
+            }
 
             unsigned int nChar = 0;
             charBuffer[nChar++] = c;
 
-            // the location of the last '[Ee]' exponent
-            unsigned int exponent = 0;
-
             while (is_.get(c))
             {
                 if (isdigit(c))
                 {
                     hasDigit = true;
                 }
-                else if (isalpha(c))
-                {
-                    notLabel = true;
-
-                    if (c == 'E' || c == 'e')
-                    {
-                        if (exponent || !hasDigit)
-                        {
-                            // mantissa had no digits,
-                            // or already saw '[Ee]' before
-                            notScalar = true;
-                        }
-
-                        // remember this location
-                        exponent = nChar;
-                    }
-                    else
-                    {
-                        notScalar = true;
-                    }
-                }
-                else if (c == '+' || c == '-')
-                {
-                    notLabel = true;
-
-                    // only allowed once in exponent
-                    if (!exponent || exponent+1 != nChar)
-                    {
-                        notScalar = true;
-                    }
-                    else
-                    {
-                        // require some digits again
-                        hasDigit = false;
-                    }
-                }
                 else if (c == '.')
                 {
-                    // notLabel means we already saw '.' or '[Ee]' before
+                    // saw '.' or '[Ee]' before
-                    // cannot have '.' again
+                    // bad position - stop parsing
-                    if (notLabel)
+                    if (floatState)
                     {
-                        notScalar = true;
+                        break;
+                    }
+                    floatState = 1;
+                }
+                else if (c == 'E' || c == 'e')
+                {
+                    // saw '[Ee]' before, or mantissa had no digits
+                    // bad position - stop parsing
+                    if (floatState > 1 || !hasDigit)
+                    {
+                        break;
+                    }
+                    floatState = 2;
+                    // require some digits again
+                    hasDigit = false;
+
+                    charBuffer[nChar++] = c;
+                    if (nChar >= sizeof(charBuffer))
+                    {
+                        // runaway argument - avoid buffer overflow
+                        t.setBad();
+                        return *this;
+                    }
+
+                    if (is_.get(c))
+                    {
+                        hasDigit = isdigit(c);
+
+                        if (!hasDigit && c != '+' && c != '-')
+                        {
+                            // not digit or [-+] - stop parsing
+                            break;
+                        }
+                    }
+                    else
+                    {
+                        break;
                     }
-                    notLabel = true;
                 }
                 else
                 {
@@ -265,11 +264,6 @@ Foam::Istream& Foam::ISstream::read(token& t)
             }
             charBuffer[nChar] = '\0';
 
-            if (!hasDigit)
-            {
-                notLabel = notScalar = true;
-            }
-
             setState(is_.rdstate());
 
             if (!is_.bad())
@@ -281,19 +275,19 @@ Foam::Istream& Foam::ISstream::read(token& t)
                     // a single '-' is punctuation
                     t = token::punctuationToken(token::SUBTRACT);
                 }
-                else if (notScalar)
+                else if (!hasDigit)
                 {
-                    // not label or scalar: must be a word
+                    // no digits is an error
-                    t = new word(charBuffer);
+                    t.setBad();
                 }
-                else if (notLabel)
+                else if (floatState)
                 {
-                    // not label: must be a scalar
+                    // scalar
                     t = scalar(atof(charBuffer));
                 }
-                else if (hasDigit)
+                else
                 {
-                    // has digit: treat as a label
+                    // label
                     long lt = atol(charBuffer);
                     t = label(lt);
 
@@ -303,11 +297,6 @@ Foam::Istream& Foam::ISstream::read(token& t)
                         t = scalar(atof(charBuffer));
                     }
                 }
-                else
-                {
-                    // some else: must be a word
-                    t = new word(charBuffer);
-                }
             }
             else
             {