Use copy-and-swap in Tokenizers

Ensures that the classes behave consistently when copied, moved, copy assigned,
and move assigned.

src/tokenizer.cpp

@@ -68,6 +68,28 @@ Tokenizer::Tokenizer(Tokenizer && rhs) :
     reset();
 }
 
+Tokenizer& Tokenizer::operator=(const Tokenizer& other)
+{
+    Tokenizer tmp(other);
+    swap(tmp);
+    return *this;
+}
+
+Tokenizer& Tokenizer::operator=(Tokenizer&& other)
+{
+    Tokenizer tmp(std::move(other));
+    swap(tmp);
+    return *this;
+}
+
+void Tokenizer::swap(Tokenizer& other)
+{
+    std::swap(text, other.text);
+    std::swap(separators, other.separators);
+    std::swap(start, other.start);
+    std::swap(ntokens, other.ntokens);
+}
+
 /*! Re-position the tokenizer state to the first word,
  * i.e. the first non-separator character */
 void Tokenizer::reset() {
@@ -181,6 +203,25 @@ ValueTokenizer::ValueTokenizer(const ValueTokenizer &rhs) : tokens(rhs.tokens) {
 ValueTokenizer::ValueTokenizer(ValueTokenizer &&rhs) : tokens(std::move(rhs.tokens)) {
 }
 
+ValueTokenizer& ValueTokenizer::operator=(const ValueTokenizer& other)
+{
+    ValueTokenizer tmp(other);
+    swap(tmp);
+    return *this;
+}
+
+ValueTokenizer& ValueTokenizer::operator=(ValueTokenizer&& other)
+{
+    ValueTokenizer tmp(std::move(other));
+    swap(tmp);
+    return *this;
+}
+
+void ValueTokenizer::swap(ValueTokenizer& other)
+{
+    std::swap(tokens, other.tokens);
+}
+
 /*! Indicate whether more tokens are available
  *
  * \return   true if there are more tokens, false if not */