From 73d4dd2d78eda1e2f8889d1913a97a60cec86876 Mon Sep 17 00:00:00 2001
From: Daniel Karbach <daniel.karbach@localhorst.tv>
Date: Tue, 17 Nov 2015 11:34:09 +0100
Subject: [PATCH] fix comment handling in TokenStreamReader

---
 src/io/Token.hpp             |   4 ++
 src/io/TokenStreamReader.hpp |   2 +
 src/io/token.cpp             | 100 +++++++++++++++++++++++++++++------
 3 files changed, 91 insertions(+), 15 deletions(-)

diff --git a/src/io/Token.hpp b/src/io/Token.hpp
index 9813eb0..e777641 100644
--- a/src/io/Token.hpp
+++ b/src/io/Token.hpp
@@ -1,6 +1,7 @@
 #ifndef BLANK_IO_TOKEN_HPP_
 #define BLANK_IO_TOKEN_HPP_
 
+#include <iosfwd>
 #include <string>
 
 
@@ -29,6 +30,9 @@ struct Token {
 	std::string value;
 };
 
+std::ostream &operator <<(std::ostream &, Token::Type);
+std::ostream &operator <<(std::ostream &, const Token &);
+
 }
 
 #endif
diff --git a/src/io/TokenStreamReader.hpp b/src/io/TokenStreamReader.hpp
index d4356aa..9436aba 100644
--- a/src/io/TokenStreamReader.hpp
+++ b/src/io/TokenStreamReader.hpp
@@ -45,6 +45,8 @@ public:
 	unsigned long GetULong();
 
 private:
+	void SkipComments();
+
 	void Assert(Token::Type);
 	Token::Type GetType() const noexcept;
 	const std::string &GetValue() const noexcept;
diff --git a/src/io/token.cpp b/src/io/token.cpp
index b462df7..07a5e98 100644
--- a/src/io/token.cpp
+++ b/src/io/token.cpp
@@ -1,8 +1,11 @@
+#include "Token.hpp"
 #include "Tokenizer.hpp"
 #include "TokenStreamReader.hpp"
 
 #include <cctype>
 #include <istream>
+#include <ostream>
+#include <sstream>
 #include <stdexcept>
 #include <glm/gtc/quaternion.hpp>
 
@@ -11,6 +14,59 @@ using namespace std;
 
 namespace blank {
 
+ostream &operator <<(ostream &out, Token::Type t) {
+	switch (t) {
+		case Token::ANGLE_BRACKET_OPEN:
+			return out << "ANGLE_BRACKET_OPEN";
+		case Token::ANGLE_BRACKET_CLOSE:
+			return out << "ANGLE_BRACKET_CLOSE";
+		case Token::CHEVRON_OPEN:
+			return out << "CHEVRON_OPEN";
+		case Token::CHEVRON_CLOSE:
+			return out << "CHEVRON_CLOSE";
+		case Token::BRACKET_OPEN:
+			return out << "BRACKET_OPEN";
+		case Token::BRACKET_CLOSE:
+			return out << "BRACKET_CLOSE";
+		case Token::PARENTHESIS_OPEN:
+			return out << "PARENTHESIS_OPEN";
+		case Token::PARENTHESIS_CLOSE:
+			return out << "PARENTHESIS_CLOSE";
+		case Token::COLON:
+			return out << "COLON";
+		case Token::SEMICOLON:
+			return out << "SEMICOLON";
+		case Token::COMMA:
+			return out << "COMMA";
+		case Token::EQUALS:
+			return out << "EQUALS";
+		case Token::NUMBER:
+			return out << "NUMBER";
+		case Token::STRING:
+			return out << "STRING";
+		case Token::IDENTIFIER:
+			return out << "IDENTIFIER";
+		case Token::COMMENT:
+			return out << "COMMENT";
+		default:
+			return out << "UNKNOWN";
+	}
+}
+
+ostream &operator <<(ostream &out, const Token &t) {
+	out << t.type;
+	switch (t.type) {
+		case Token::UNKNOWN:
+		case Token::NUMBER:
+		case Token::STRING:
+		case Token::IDENTIFIER:
+		case Token::COMMENT:
+			return out << '(' << t.value << ')';
+		default:
+			return out;
+	}
+}
+
 Tokenizer::Tokenizer(istream &in)
 : in(in)
 , current() {
@@ -195,27 +251,35 @@ bool TokenStreamReader::HasMore() {
 	if (cached) {
 		return true;
 	}
-	while (in.HasMore()) {
-		if (in.Next().type != Token::COMMENT) {
-			cached = true;
-			return true;
-		}
-	}
-	return false;
+	SkipComments();
+	return cached;
 }
 
 const Token &TokenStreamReader::Next() {
+	SkipComments();
+	cached = false;
+	return in.Current();
+}
+
+void TokenStreamReader::SkipComments() {
 	if (cached) {
-		cached = false;
-		return in.Current();
-	} else {
-		return in.Next();
+		if (in.Current().type == Token::COMMENT) {
+			cached = false;
+		} else {
+			return;
+		}
+	}
+	while (in.HasMore()) {
+		if (in.Next().type != Token::COMMENT) {
+			cached = true;
+			return;
+		}
 	}
 }
 
 const Token &TokenStreamReader::Peek() {
 	if (!cached) {
-		in.Next();
+		Next();
 		cached = true;
 	}
 	return in.Current();
@@ -224,7 +288,9 @@ const Token &TokenStreamReader::Peek() {
 
 void TokenStreamReader::Assert(Token::Type t) {
 	if (GetType() != t) {
-		throw runtime_error("unexpected token in input stream");
+		stringstream s;
+		s << "unexpected token in input stream: expected " << t << ", but got " << in.Current();
+		throw runtime_error(s.str());
 	}
 }
 
@@ -356,10 +422,14 @@ bool TokenStreamReader::GetBool() {
 			} else if (GetValue() == "false" || GetValue() == "no" || GetValue() == "off") {
 				return false;
 			} else {
-				throw runtime_error("unexpected value in input stream");
+				throw runtime_error("unexpected value in input stream: cannot cast " + GetValue() + " to bool");
 			}
 		default:
-			throw runtime_error("unexpected token in input stream");
+			{
+				stringstream s;
+				s << "unexpected token in input stream: cannot cast " << in.Current() << " to bool";
+				throw runtime_error(s.str());
+			}
 	}
 }
 
-- 
2.39.2