From 7ac0f869efc7e6528efea4e31f0f1daab0fc83f5 Mon Sep 17 00:00:00 2001
From: Richard Berger <richard.berger@temple.edu>
Date: Mon, 18 May 2020 16:40:11 -0400
Subject: [PATCH] Add ValueTokenizer

---
 src/tokenizer.cpp                 | 80 +++++++++++++++++++++++++++++++
 src/tokenizer.h                   | 48 +++++++++++++++++++
 unittest/utils/test_tokenizer.cpp | 40 ++++++++++++++++
 3 files changed, 168 insertions(+)

diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
index 1046465956..ca410f275a 100644
--- a/src/tokenizer.cpp
+++ b/src/tokenizer.cpp
@@ -16,6 +16,7 @@
 ------------------------------------------------------------------------- */
 
 #include "tokenizer.h"
+#include "utils.h"
 
 using namespace LAMMPS_NS;
 
@@ -44,6 +45,14 @@ Tokenizer::iterator Tokenizer::end() {
     return tokens.end();
 }
 
+Tokenizer::const_iterator Tokenizer::cbegin() const {
+    return tokens.cbegin();
+}
+
+Tokenizer::const_iterator Tokenizer::cend() const {
+    return tokens.cend();
+}
+
 const std::string & Tokenizer::operator[](size_t index) {
     return tokens[index];
 }
@@ -51,3 +60,74 @@ const std::string & Tokenizer::operator[](size_t index) {
 const size_t Tokenizer::count() const {
     return tokens.size();
 }
+
+
+ValueTokenizer::ValueTokenizer(const std::string & str, const std::string & seperators) : tokens(str, seperators) {
+    current  = tokens.begin();
+}
+
+bool ValueTokenizer::has_next() const {
+    return current != tokens.cend();
+}
+
+std::string ValueTokenizer::next_string() {
+    if (has_next()) {
+        std::string value = *current;
+        ++current;
+        return value;
+    }
+    return "";
+}
+
+int ValueTokenizer::next_int() {
+    if (has_next()) {
+        if(!utils::is_integer(*current)) {
+            throw InvalidIntegerException(*current);
+        }
+        int value = atoi(current->c_str());
+        ++current;
+        return value;
+    }
+    return 0;
+}
+
+bigint ValueTokenizer::next_bigint() {
+    if (has_next()) {
+        if(!utils::is_integer(*current)) {
+            throw InvalidIntegerException(*current);
+        }
+        bigint value = ATOBIGINT(current->c_str());
+        ++current;
+        return value;
+    }
+    return 0;
+}
+
+tagint ValueTokenizer::next_tagint() {
+    if (current != tokens.end()) {
+        if(!utils::is_integer(*current)) {
+            throw InvalidIntegerException(*current);
+        }
+        tagint value = ATOTAGINT(current->c_str());
+        ++current;
+        return value;
+    }
+    return 0;
+}
+
+double ValueTokenizer::next_double() {
+    if (current != tokens.end()) {
+        if(!utils::is_double(*current)) {
+            throw InvalidFloatException(*current);
+        }
+
+        double value = atof(current->c_str());
+        ++current;
+        return value;
+    }
+    return 0.0;
+}
+
+void ValueTokenizer::skip(int ntokens) {
+    current = std::next(current, ntokens);
+}
diff --git a/src/tokenizer.h b/src/tokenizer.h
index 855b21d7f2..d35107ad3a 100644
--- a/src/tokenizer.h
+++ b/src/tokenizer.h
@@ -20,6 +20,8 @@
 
 #include <string>
 #include <vector>
+#include "lmptype.h"
+#include <exception>
 
 namespace LAMMPS_NS {
 
@@ -27,16 +29,62 @@ class Tokenizer {
     std::vector<std::string> tokens;
 public:
     typedef std::vector<std::string>::iterator iterator;
+    typedef std::vector<std::string>::const_iterator const_iterator;
 
     Tokenizer(const std::string & str, const std::string & seperators = " \t\r\n\f");
 
     iterator begin();
     iterator end();
+    const_iterator cbegin() const;
+    const_iterator cend() const;
 
     const std::string & operator[](size_t index);
     const size_t count() const;
 };
 
+class TokenizerException : public std::exception {
+  std::string message;
+public:
+  TokenizerException(const std::string & msg, const std::string & token) : message(msg + ": '" + token + "'") {
+  }
+
+  ~TokenizerException() throw() {
+  }
+
+  virtual const char * what() const throw() {
+    return message.c_str();
+  }
+};
+
+class InvalidIntegerException : public TokenizerException {
+public:
+    InvalidIntegerException(const std::string & token) : TokenizerException("Not a valid integer number", token) {
+    }
+};
+
+class InvalidFloatException : public TokenizerException {
+public:
+    InvalidFloatException(const std::string & token) : TokenizerException("Not a valid floating-point number", token) {
+    }
+};
+
+class ValueTokenizer {
+    Tokenizer tokens;
+    Tokenizer::const_iterator current;
+public:
+    ValueTokenizer(const std::string & str, const std::string & seperators = " \t\r\n\f");
+
+    std::string next_string();
+    tagint next_tagint();
+    bigint next_bigint();
+    int    next_int();
+    double next_double();
+
+    bool has_next() const;
+    void skip(int ntokens);
+};
+
+
 }
 
 #endif
diff --git a/unittest/utils/test_tokenizer.cpp b/unittest/utils/test_tokenizer.cpp
index 86da541c9e..23f2e85d98 100644
--- a/unittest/utils/test_tokenizer.cpp
+++ b/unittest/utils/test_tokenizer.cpp
@@ -59,3 +59,43 @@ TEST(Tokenizer, for_loop) {
     ASSERT_THAT(list[0], Eq("test"));
     ASSERT_THAT(list[1], Eq("word"));
 }
+
+TEST(ValueTokenizer, empty_string) {
+    ValueTokenizer values("");
+    ASSERT_FALSE(values.has_next());
+}
+
+TEST(ValueTokenizer, bad_integer) {
+    ValueTokenizer values("f10");
+    ASSERT_THROW(values.next_int(), InvalidIntegerException);
+}
+
+TEST(ValueTokenizer, bad_double) {
+    ValueTokenizer values("1a.0");
+    ASSERT_THROW(values.next_double(), InvalidFloatException);
+}
+
+TEST(ValueTokenizer, valid_int) {
+    ValueTokenizer values("10");
+    ASSERT_EQ(values.next_int(), 10);
+}
+
+TEST(ValueTokenizer, valid_tagint) {
+    ValueTokenizer values("42");
+    ASSERT_EQ(values.next_tagint(), 42);
+}
+
+TEST(ValueTokenizer, valid_bigint) {
+    ValueTokenizer values("42");
+    ASSERT_EQ(values.next_bigint(), 42);
+}
+
+TEST(ValueTokenizer, valid_double) {
+    ValueTokenizer values("3.14");
+    ASSERT_DOUBLE_EQ(values.next_double(), 3.14);
+}
+
+TEST(ValueTokenizer, valid_double_with_exponential) {
+    ValueTokenizer values("3.14e22");
+    ASSERT_DOUBLE_EQ(values.next_double(), 3.14e22);
+}