fix comment handling in TokenStreamReader

diff --git a/src/io/Token.hpp b/src/io/Token.hpp
index 9813eb0adeb2f3a31f64e5086bc154b58f2822b4..e7776411874bf4c0f960d6613a6541068d63a5cc 100644 (file)
--- a/src/io/Token.hpp
+++ b/src/io/Token.hpp
@@ -1,6 +1,7 @@
 #ifndef BLANK_IO_TOKEN_HPP_
 #define BLANK_IO_TOKEN_HPP_
 
+#include <iosfwd>
 #include <string>
 
 
@@ -29,6 +30,9 @@ struct Token {
        std::string value;
 };
 
+std::ostream &operator <<(std::ostream &, Token::Type);
+std::ostream &operator <<(std::ostream &, const Token &);
+
 }
 
 #endif

diff --git a/src/io/TokenStreamReader.hpp b/src/io/TokenStreamReader.hpp
index d4356aa3d91d0e7129ab8ea3353f545f1544d004..9436aba3f3dc355a3d8d050c102d43a4d8e42ed9 100644 (file)
--- a/src/io/TokenStreamReader.hpp
+++ b/src/io/TokenStreamReader.hpp
@@ -45,6 +45,8 @@ public:
        unsigned long GetULong();
 
 private:
+       void SkipComments();
+
        void Assert(Token::Type);
        Token::Type GetType() const noexcept;
        const std::string &GetValue() const noexcept;

diff --git a/src/io/token.cpp b/src/io/token.cpp
index b462df778b593ee11927d6e80da46031b530b709..07a5e981cd889d01d5920797ee0a4d63dd0ff604 100644 (file)
--- a/src/io/token.cpp
+++ b/src/io/token.cpp
@@ -1,8 +1,11 @@
+#include "Token.hpp"
 #include "Tokenizer.hpp"
 #include "TokenStreamReader.hpp"
 
 #include <cctype>
 #include <istream>
+#include <ostream>
+#include <sstream>
 #include <stdexcept>
 #include <glm/gtc/quaternion.hpp>
 
@@ -11,6 +14,59 @@ using namespace std;
 
 namespace blank {
 
+ostream &operator <<(ostream &out, Token::Type t) {
+       switch (t) {
+               case Token::ANGLE_BRACKET_OPEN:
+                       return out << "ANGLE_BRACKET_OPEN";
+               case Token::ANGLE_BRACKET_CLOSE:
+                       return out << "ANGLE_BRACKET_CLOSE";
+               case Token::CHEVRON_OPEN:
+                       return out << "CHEVRON_OPEN";
+               case Token::CHEVRON_CLOSE:
+                       return out << "CHEVRON_CLOSE";
+               case Token::BRACKET_OPEN:
+                       return out << "BRACKET_OPEN";
+               case Token::BRACKET_CLOSE:
+                       return out << "BRACKET_CLOSE";
+               case Token::PARENTHESIS_OPEN:
+                       return out << "PARENTHESIS_OPEN";
+               case Token::PARENTHESIS_CLOSE:
+                       return out << "PARENTHESIS_CLOSE";
+               case Token::COLON:
+                       return out << "COLON";
+               case Token::SEMICOLON:
+                       return out << "SEMICOLON";
+               case Token::COMMA:
+                       return out << "COMMA";
+               case Token::EQUALS:
+                       return out << "EQUALS";
+               case Token::NUMBER:
+                       return out << "NUMBER";
+               case Token::STRING:
+                       return out << "STRING";
+               case Token::IDENTIFIER:
+                       return out << "IDENTIFIER";
+               case Token::COMMENT:
+                       return out << "COMMENT";
+               default:
+                       return out << "UNKNOWN";
+       }
+}
+
+ostream &operator <<(ostream &out, const Token &t) {
+       out << t.type;
+       switch (t.type) {
+               case Token::UNKNOWN:
+               case Token::NUMBER:
+               case Token::STRING:
+               case Token::IDENTIFIER:
+               case Token::COMMENT:
+                       return out << '(' << t.value << ')';
+               default:
+                       return out;
+       }
+}
+
 Tokenizer::Tokenizer(istream &in)
 : in(in)
 , current() {
@@ -195,27 +251,35 @@ bool TokenStreamReader::HasMore() {
        if (cached) {
                return true;
        }
-       while (in.HasMore()) {
-               if (in.Next().type != Token::COMMENT) {
-                       cached = true;
-                       return true;
-               }
-       }
-       return false;
+       SkipComments();
+       return cached;
 }
 
 const Token &TokenStreamReader::Next() {
+       SkipComments();
+       cached = false;
+       return in.Current();
+}
+
+void TokenStreamReader::SkipComments() {
        if (cached) {
-               cached = false;
-               return in.Current();
-       } else {
-               return in.Next();
+               if (in.Current().type == Token::COMMENT) {
+                       cached = false;
+               } else {
+                       return;
+               }
+       }
+       while (in.HasMore()) {
+               if (in.Next().type != Token::COMMENT) {
+                       cached = true;
+                       return;
+               }
        }
 }
 
 const Token &TokenStreamReader::Peek() {
        if (!cached) {
-               in.Next();
+               Next();
                cached = true;
        }
        return in.Current();
@@ -224,7 +288,9 @@ const Token &TokenStreamReader::Peek() {
 
 void TokenStreamReader::Assert(Token::Type t) {
        if (GetType() != t) {
-               throw runtime_error("unexpected token in input stream");
+               stringstream s;
+               s << "unexpected token in input stream: expected " << t << ", but got " << in.Current();
+               throw runtime_error(s.str());
        }
 }
 
@@ -356,10 +422,14 @@ bool TokenStreamReader::GetBool() {
                        } else if (GetValue() == "false" || GetValue() == "no" || GetValue() == "off") {
                                return false;
                        } else {
-                               throw runtime_error("unexpected value in input stream");
+                               throw runtime_error("unexpected value in input stream: cannot cast " + GetValue() + " to bool");
                        }
                default:
-                       throw runtime_error("unexpected token in input stream");
+                       {
+                               stringstream s;
+                               s << "unexpected token in input stream: cannot cast " << in.Current() << " to bool";
+                               throw runtime_error(s.str());
+                       }
        }
 }