From d5e57ac02d17d9b0794127c01245214b3a708f2c Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Mon, 29 Jul 2024 23:49:13 -0400 Subject: [PATCH] reimplement using C++11 --- src/tokenizer.cpp | 93 ++++++++++++++++++++----------- unittest/utils/test_tokenizer.cpp | 34 +++++++---- 2 files changed, 82 insertions(+), 45 deletions(-) diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp index 2db7f1dd41..a2b95ab89e 100644 --- a/src/tokenizer.cpp +++ b/src/tokenizer.cpp @@ -287,15 +287,20 @@ std::string ValueTokenizer::next_string() int ValueTokenizer::next_int() { std::string current = tokens.next(); - if (!utils::is_integer(current)) { throw InvalidIntegerException(current); } - const char *str = current.c_str(); - char *end = nullptr; - auto val = std::strtoll(str, &end, 10); - // only partially converted - if ((str + current.size()) != end) { throw InvalidIntegerException(current); } - // out of range - if ((val < -MAXSMALLINT) || (val > MAXSMALLINT)) { throw InvalidIntegerException(current); } - return (int) val; + try { + std::size_t end; + auto val = std::stoi(current, &end); + // only partially converted + if (current.size() != end) { throw InvalidIntegerException(current); } + return val; + + // rethrow exceptions from std::stoi() + } catch (std::out_of_range const &) { + throw InvalidIntegerException(current); + } catch (std::invalid_argument const &) { + throw InvalidIntegerException(current); + } + return 0; } /*! Retrieve next token and convert to bigint @@ -304,15 +309,22 @@ int ValueTokenizer::next_int() bigint ValueTokenizer::next_bigint() { std::string current = tokens.next(); - if (!utils::is_integer(current)) { throw InvalidIntegerException(current); } - const char *str = current.c_str(); - char *end = nullptr; - auto val = std::strtoll(str, &end, 10); - // only partially converted - if ((str + current.size()) != end) { throw InvalidIntegerException(current); } - // out of range - if ((val < -MAXBIGINT) || (val > MAXBIGINT)) { throw InvalidIntegerException(current); } - return (bigint) val; + try { + std::size_t end; + auto val = std::stoll(current, &end, 10); + // only partially converted + if (current.size() != end) { throw InvalidIntegerException(current); } + // out of range + if ((val < (-MAXBIGINT - 1) || (val > MAXBIGINT))) { throw InvalidIntegerException(current); }; + return (bigint) val; + + // rethrow exceptions from std::stoll() + } catch (std::out_of_range const &) { + throw InvalidIntegerException(current); + } catch (std::invalid_argument const &) { + throw InvalidIntegerException(current); + } + return 0; } /*! Retrieve next token and convert to tagint @@ -321,15 +333,22 @@ bigint ValueTokenizer::next_bigint() tagint ValueTokenizer::next_tagint() { std::string current = tokens.next(); - if (!utils::is_integer(current)) { throw InvalidIntegerException(current); } - const char *str = current.c_str(); - char *end = nullptr; - auto val = std::strtoll(str, &end, 10); - // only partially converted - if ((str + current.size()) != end) { throw InvalidIntegerException(current); } - // out of range - if ((val < -MAXTAGINT) || (val > MAXTAGINT)) { throw InvalidIntegerException(current); } - return (tagint) val; + try { + std::size_t end; + auto val = std::stoll(current, &end, 10); + // only partially converted + if (current.size() != end) { throw InvalidIntegerException(current); } + // out of range + if ((val < (-MAXTAGINT - 1) || (val > MAXTAGINT))) { throw InvalidIntegerException(current); } + return (tagint) val; + + // rethrow exceptions from std::stoll() + } catch (std::out_of_range const &) { + throw InvalidIntegerException(current); + } catch (std::invalid_argument const &) { + throw InvalidIntegerException(current); + } + return 0; } /*! Retrieve next token and convert to double @@ -338,13 +357,19 @@ tagint ValueTokenizer::next_tagint() double ValueTokenizer::next_double() { std::string current = tokens.next(); - if (!utils::is_double(current)) { throw InvalidFloatException(current); } - const char *str = current.c_str(); - char *end = nullptr; - double val = std::strtod(str, &end); - // only partially converted - if ((str + current.size()) != end) { throw InvalidFloatException(current); } - return val; + try { + std::size_t end; + auto val = std::stod(current, &end); + // only partially converted + if (current.size() != end) { throw InvalidFloatException(current); } + return val; + // rethrow exceptions from std::stod() + } catch (std::out_of_range const &) { + throw InvalidFloatException(current); + } catch (std::invalid_argument const &) { + throw InvalidFloatException(current); + } + return 0.0; } /*! Skip over a given number of tokens diff --git a/unittest/utils/test_tokenizer.cpp b/unittest/utils/test_tokenizer.cpp index 6b4e182cdb..d5d6955b32 100644 --- a/unittest/utils/test_tokenizer.cpp +++ b/unittest/utils/test_tokenizer.cpp @@ -337,52 +337,64 @@ TEST(ValueTokenizer, move_assignment) TEST(ValueTokenizer, bad_integer) { - ValueTokenizer values("f10 f11 f12"); + ValueTokenizer values("f10 f11 f12 0xff 109951162777 " + "36893488147419103232 36893488147419103232"); ASSERT_THROW(values.next_int(), InvalidIntegerException); ASSERT_THROW(values.next_bigint(), InvalidIntegerException); ASSERT_THROW(values.next_tagint(), InvalidIntegerException); + ASSERT_THROW(values.next_int(), InvalidIntegerException); + ASSERT_THROW(values.next_int(), InvalidIntegerException); + ASSERT_THROW(values.next_tagint(), InvalidIntegerException); + ASSERT_THROW(values.next_bigint(), InvalidIntegerException); } TEST(ValueTokenizer, bad_double) { - ValueTokenizer values("1a.0"); + ValueTokenizer values("1a.0 --2.0 2.4d3 -1e20000 1.0e-1.0"); + ASSERT_THROW(values.next_double(), InvalidFloatException); + ASSERT_THROW(values.next_double(), InvalidFloatException); + ASSERT_THROW(values.next_double(), InvalidFloatException); + ASSERT_THROW(values.next_double(), InvalidFloatException); ASSERT_THROW(values.next_double(), InvalidFloatException); } TEST(ValueTokenizer, valid_int) { - ValueTokenizer values(fmt::format("10 -{} {}", MAXSMALLINT, MAXSMALLINT)); + ValueTokenizer values(fmt::format("10 {} {}", -MAXSMALLINT - 1, MAXSMALLINT)); ASSERT_EQ(values.next_int(), 10); - ASSERT_EQ(values.next_int(), -MAXSMALLINT); + ASSERT_EQ(values.next_int(), -MAXSMALLINT - 1); ASSERT_EQ(values.next_int(), MAXSMALLINT); } TEST(ValueTokenizer, valid_tagint) { - ValueTokenizer values(fmt::format("42 -{} {} -{} {}", MAXSMALLINT, MAXSMALLINT, MAXTAGINT, MAXTAGINT)); + ValueTokenizer values( + fmt::format("42 {} {} {} {}", -MAXSMALLINT - 1, MAXSMALLINT, -MAXTAGINT - 1, MAXTAGINT)); ASSERT_EQ(values.next_tagint(), 42); - ASSERT_EQ(values.next_tagint(), -MAXSMALLINT); + ASSERT_EQ(values.next_tagint(), -MAXSMALLINT - 1); ASSERT_EQ(values.next_tagint(), MAXSMALLINT); - ASSERT_EQ(values.next_tagint(), -MAXTAGINT); + ASSERT_EQ(values.next_tagint(), -MAXTAGINT - 1); ASSERT_EQ(values.next_tagint(), MAXTAGINT); } TEST(ValueTokenizer, valid_bigint) { - ValueTokenizer values(fmt::format("42 -{} {} -{} {}", MAXSMALLINT, MAXSMALLINT, MAXBIGINT, MAXBIGINT)); + ValueTokenizer values( + fmt::format("42 {} {} {} {}", -MAXSMALLINT - 1, MAXSMALLINT, -MAXBIGINT - 1, MAXBIGINT)); ASSERT_EQ(values.next_bigint(), 42); - ASSERT_EQ(values.next_bigint(), -MAXSMALLINT); + ASSERT_EQ(values.next_bigint(), -MAXSMALLINT - 1); ASSERT_EQ(values.next_bigint(), MAXSMALLINT); - ASSERT_EQ(values.next_bigint(), -MAXBIGINT); + ASSERT_EQ(values.next_bigint(), -MAXBIGINT - 1); ASSERT_EQ(values.next_bigint(), MAXBIGINT); } TEST(ValueTokenizer, valid_double) { - ValueTokenizer values("3.14 -0.00002 .1 " + std::to_string(MAXBIGINT)); + ValueTokenizer values("3.14 -0.00002 .1 0xff " + std::to_string(MAXBIGINT)); ASSERT_DOUBLE_EQ(values.next_double(), 3.14); ASSERT_DOUBLE_EQ(values.next_double(), -0.00002); ASSERT_DOUBLE_EQ(values.next_double(), 0.1); + ASSERT_DOUBLE_EQ(values.next_double(), 255); ASSERT_DOUBLE_EQ(values.next_double(), MAXBIGINT); }