From a6a2879807254e1fd0efd274a276114eea57f72e Mon Sep 17 00:00:00 2001
From: Axel Kohlmeyer <akohlmey@gmail.com>
Date: Mon, 15 Mar 2021 12:02:58 -0400
Subject: [PATCH] add a few more tokenizer test cases for added use cases.

---
 unittest/utils/test_tokenizer.cpp | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/unittest/utils/test_tokenizer.cpp b/unittest/utils/test_tokenizer.cpp
index a7cf33f8d3..de7472599c 100644
--- a/unittest/utils/test_tokenizer.cpp
+++ b/unittest/utils/test_tokenizer.cpp
@@ -102,7 +102,7 @@ TEST(Tokenizer, default_separators)
     ASSERT_EQ(t.count(), 2);
 }
 
-TEST(Tokenizer, as_vector)
+TEST(Tokenizer, as_vector1)
 {
     Tokenizer t(" \r\n test \t word \f");
     std::vector<std::string> list = t.as_vector();
@@ -110,6 +110,29 @@ TEST(Tokenizer, as_vector)
     ASSERT_THAT(list[1], Eq("word"));
 }
 
+TEST(Tokenizer, as_vector2)
+{
+    auto list = Tokenizer("a\\b\\c","\\").as_vector();
+    ASSERT_THAT(list[0], Eq("a"));
+    ASSERT_THAT(list[1], Eq("b"));
+    ASSERT_THAT(list[2], Eq("c"));
+    ASSERT_EQ(list.size(), 3);
+}
+
+TEST(Tokenizer, as_vector3)
+{
+    auto list = Tokenizer ("a\\","\\").as_vector();
+    ASSERT_THAT(list[0], Eq("a"));
+    ASSERT_EQ(list.size(), 1);
+}
+
+TEST(Tokenizer, as_vector4)
+{
+    auto list = Tokenizer ("\\a","\\").as_vector();
+    ASSERT_THAT(list[0], Eq("a"));
+    ASSERT_EQ(list.size(), 1);
+}
+
 TEST(ValueTokenizer, empty_string)
 {
     ValueTokenizer values("");