diff --git a/applications/test/tokenizeTest/Make/files b/applications/test/tokenizeTest/Make/files new file mode 100644 index 0000000000..c2db390007 --- /dev/null +++ b/applications/test/tokenizeTest/Make/files @@ -0,0 +1,3 @@ +tokenizeTest.C + +EXE = $(FOAM_USER_APPBIN)/tokenizeTest diff --git a/applications/test/tokenizeTest/Make/options b/applications/test/tokenizeTest/Make/options new file mode 100644 index 0000000000..42da6e3418 --- /dev/null +++ b/applications/test/tokenizeTest/Make/options @@ -0,0 +1,7 @@ +/* + check for consistent behaviour with non-optimized code + */ + +EXE_INC = \ + -DFULLDEBUG -g -O0 + diff --git a/applications/test/tokenizeTest/tokenizeTest.C b/applications/test/tokenizeTest/tokenizeTest.C new file mode 100644 index 0000000000..deb675ee00 --- /dev/null +++ b/applications/test/tokenizeTest/tokenizeTest.C @@ -0,0 +1,87 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | OpenFOAM: The Open Source CFD Toolbox + \\ / O peration | + \\ / A nd | Copyright (C) 1991-2009 OpenCFD Ltd. + \\/ M anipulation | +------------------------------------------------------------------------------- +License + This file is part of OpenFOAM. + + OpenFOAM is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the + Free Software Foundation; either version 2 of the License, or (at your + option) any later version. + + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with OpenFOAM; if not, write to the Free Software Foundation, + Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +Application + +Description + Test the tokenizing of various things +\*---------------------------------------------------------------------------*/ + +#include "argList.H" +#include "IOobject.H" +#include "IOstreams.H" +#include "IFstream.H" +#include "IStringStream.H" + +using namespace Foam; + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // +// Main program: + +int main(int argc, char *argv[]) +{ + argList::noParallel(); + argList::validArgs.insert("string .. stringN"); + argList::validOptions.insert("file", "name"); + + argList args(argc, argv, false, true); + + forAll(args.additionalArgs(), argI) + { + const string& rawArg = args.additionalArgs()[argI]; + Info<< "input string: " << rawArg << nl; + + IStringStream is(rawArg); + + while (is.good()) + { + token tok(is); + Info<< "token: " << tok.info() << endl; + } + + Info<< nl; + IOobject::writeDivider(Info); + } + + + if (args.optionFound("file")) + { + IFstream is(args.option("file")); + + Info<< "tokenizing file: " << args.option("file") << nl; + + while (is.good()) + { + token tok(is); + Info<< "token: " << tok.info() << endl; + } + + Info<< nl; + IOobject::writeDivider(Info); + } + + return 0; +} + +// ************************************************************************* // diff --git a/src/OpenFOAM/db/IOstreams/Sstreams/ISstream.C b/src/OpenFOAM/db/IOstreams/Sstreams/ISstream.C index ba7635dfcd..95de958c90 100644 --- a/src/OpenFOAM/db/IOstreams/Sstreams/ISstream.C +++ b/src/OpenFOAM/db/IOstreams/Sstreams/ISstream.C @@ -176,7 +176,7 @@ Foam::Istream& Foam::ISstream::read(token& t) // // ideally match the equivalent of this regular expression // - // /^[-+]?([0-9]+\.?[0-9]*|\.[0-9]+)([Ee][-+]?[0-9]+)?$/ + // /[-+]?([0-9]+\.?[0-9]*|\.[0-9]+)([Ee][-+]?[0-9]+)?/ // case '-' : case '.' : @@ -186,69 +186,68 @@ Foam::Istream& Foam::ISstream::read(token& t) // has a digit bool hasDigit = isdigit(c); - // has contents that cannot be label - bool notLabel = (c == '.'); - - // has contents that cannot be scalar - bool notScalar = false; + // 0 = before a decimal + // 1 = at/after a decimal + // 2 = in exponent + int floatState = 0; + if (c == '.') + { + floatState = 1; + } unsigned int nChar = 0; charBuffer[nChar++] = c; - // the location of the last '[Ee]' exponent - unsigned int exponent = 0; - while (is_.get(c)) { if (isdigit(c)) { hasDigit = true; } - else if (isalpha(c)) - { - notLabel = true; - - if (c == 'E' || c == 'e') - { - if (exponent || !hasDigit) - { - // mantissa had no digits, - // or already saw '[Ee]' before - notScalar = true; - } - - // remember this location - exponent = nChar; - } - else - { - notScalar = true; - } - } - else if (c == '+' || c == '-') - { - notLabel = true; - - // only allowed once in exponent - if (!exponent || exponent+1 != nChar) - { - notScalar = true; - } - else - { - // require some digits again - hasDigit = false; - } - } else if (c == '.') { - // notLabel means we already saw '.' or '[Ee]' before - // cannot have '.' again - if (notLabel) + // saw '.' or '[Ee]' before + // bad position - stop parsing + if (floatState) { - notScalar = true; + break; + } + floatState = 1; + } + else if (c == 'E' || c == 'e') + { + // saw '[Ee]' before, or mantissa had no digits + // bad position - stop parsing + if (floatState > 1 || !hasDigit) + { + break; + } + floatState = 2; + // require some digits again + hasDigit = false; + + charBuffer[nChar++] = c; + if (nChar >= sizeof(charBuffer)) + { + // runaway argument - avoid buffer overflow + t.setBad(); + return *this; + } + + if (is_.get(c)) + { + hasDigit = isdigit(c); + + if (!hasDigit && c != '+' && c != '-') + { + // not digit or [-+] - stop parsing + break; + } + } + else + { + break; } - notLabel = true; } else { @@ -265,11 +264,6 @@ Foam::Istream& Foam::ISstream::read(token& t) } charBuffer[nChar] = '\0'; - if (!hasDigit) - { - notLabel = notScalar = true; - } - setState(is_.rdstate()); if (!is_.bad()) @@ -281,19 +275,19 @@ Foam::Istream& Foam::ISstream::read(token& t) // a single '-' is punctuation t = token::punctuationToken(token::SUBTRACT); } - else if (notScalar) + else if (!hasDigit) { - // not label or scalar: must be a word - t = new word(charBuffer); + // no digits is an error + t.setBad(); } - else if (notLabel) + else if (floatState) { - // not label: must be a scalar + // scalar t = scalar(atof(charBuffer)); } - else if (hasDigit) + else { - // has digit: treat as a label + // label long lt = atol(charBuffer); t = label(lt); @@ -303,11 +297,6 @@ Foam::Istream& Foam::ISstream::read(token& t) t = scalar(atof(charBuffer)); } } - else - { - // some else: must be a word - t = new word(charBuffer); - } } else {