complete documentation of tokenizer classes

src/tokenizer.cpp

@@ -30,6 +30,13 @@ TokenizerException::TokenizerException(const std::string & msg, const std::strin
     }
 }
 
+/** Class for splitting text into words
+ *
+ * This tokenizer will break down a string into sub-strings (i.e words)
+ * separated by the given separator characters.
+ *
+ * \sa LAMMPS_NS::ValueTokenizer TokenizerException */
+
 Tokenizer::Tokenizer(const std::string & str, const std::string & separators) :
     text(str), separators(separators), start(0), ntokens(std::string::npos)
 {
@@ -48,14 +55,23 @@ Tokenizer::Tokenizer(Tokenizer && rhs) :
     reset();
 }
 
+/*! Reposition the tokenizer state to the first word,
+ * i.e. the first non-separator character */
 void Tokenizer::reset() {
     start = text.find_first_not_of(separators);
 }
 
+/*! Search the text to be processed for a sub-string.
+ *
+ * \param  str  string to be searched for
+ * \return      true if string was found, false if not */
 bool Tokenizer::contains(const std::string & str) const {
     return text.find(str) != std::string::npos;
 }
 
+/*! Skip over a given number of tokens
+ *
+ * \param  n  number of tokens to skip over */
 void Tokenizer::skip(int n) {
     for(int i = 0; i < n; ++i) {
         if(!has_next()) throw TokenizerException("No more tokens", "");
@@ -70,10 +86,16 @@ void Tokenizer::skip(int n) {
     }
 }
 
+/*! Indicate whether more tokens are available
+ *
+ * \return   true if there are more tokens, false if not */
 bool Tokenizer::has_next() const {
     return start != std::string::npos;
 }
 
+/*! Retrieve next token.
+ *
+ * \return   string with the next token */
 std::string Tokenizer::next() {
     if(!has_next()) throw TokenizerException("No more tokens", "");
 
@@ -90,6 +112,9 @@ std::string Tokenizer::next() {
     return token;
 }
 
+/*! Count number of tokens in text.
+ *
+ * \return   number of counted tokens */
 size_t Tokenizer::count() {
     // lazy evaluation
     if (ntokens == std::string::npos) {
@@ -98,6 +123,9 @@ size_t Tokenizer::count() {
     return ntokens;
 }
 
+/*! Retrieve the entire text converted to an STL vector of tokens.
+ *
+ * \return   The STL vector */
 std::vector<std::string> Tokenizer::as_vector() {
   // store current state
   size_t current = start;
@@ -117,6 +145,9 @@ std::vector<std::string> Tokenizer::as_vector() {
   return tokens;
 }
 
+/*! Class for reading text with numbers
+ *
+ * \sa LAMMPS_NS::Tokenizer InvalidIntegerException InvalidFloatException */
 
 ValueTokenizer::ValueTokenizer(const std::string & str, const std::string & separators) : tokens(str, separators) {
 }
@@ -127,14 +158,24 @@ ValueTokenizer::ValueTokenizer(const ValueTokenizer & rhs) : tokens(rhs.tokens)
 ValueTokenizer::ValueTokenizer(ValueTokenizer && rhs) : tokens(std::move(rhs.tokens)) {
 }
 
+/*! Indicate whether more tokens are available
+ *
+ * \return   true if there are more tokens, false if not */
 bool ValueTokenizer::has_next() const {
     return tokens.has_next();
 }
 
+/*! Search the text to be processed for a sub-string.
+ *
+ * \param  value  string with value to be searched for
+ * \return        true if string was found, false if not */
 bool ValueTokenizer::contains(const std::string & value) const {
     return tokens.contains(value);
 }
 
+/*! Retrieve next token
+ *
+ * \return   string with next token */
 std::string ValueTokenizer::next_string() {
     if (has_next()) {
         std::string value = tokens.next();
@@ -143,6 +184,9 @@ std::string ValueTokenizer::next_string() {
     return "";
 }
 
+/*! Retrieve next token and convert to int
+ *
+ * \return   value of next token */
 int ValueTokenizer::next_int() {
     if (has_next()) {
         std::string current = tokens.next();
@@ -155,6 +199,9 @@ int ValueTokenizer::next_int() {
     return 0;
 }
 
+/*! Retrieve next token and convert to bigint
+ *
+ * \return   value of next token */
 bigint ValueTokenizer::next_bigint() {
     if (has_next()) {
         std::string current = tokens.next();
@@ -167,6 +214,9 @@ bigint ValueTokenizer::next_bigint() {
     return 0;
 }
 
+/*! Retrieve next token and convert to tagint
+ *
+ * \return   value of next token */
 tagint ValueTokenizer::next_tagint() {
     if (has_next()) {
         std::string current = tokens.next();
@@ -179,6 +229,9 @@ tagint ValueTokenizer::next_tagint() {
     return 0;
 }
 
+/*! Retrieve next token and convert to double
+ *
+ * \return   value of next token */
 double ValueTokenizer::next_double() {
     if (has_next()) {
         std::string current = tokens.next();
@@ -191,10 +244,16 @@ double ValueTokenizer::next_double() {
     return 0.0;
 }
 
+/*! Skip over a given number of tokens
+ *
+ * \param  n  number of tokens to skip over */
 void ValueTokenizer::skip(int n) {
     tokens.skip(n);
 }
 
+/*! Count number of tokens in text.
+ *
+ * \return   number of counted tokens */
 size_t ValueTokenizer::count() {
     return tokens.count();
 }