From 51e0a94d0e0a3bc1a4664aa9af1f20910f55201c Mon Sep 17 00:00:00 2001
From: Daniel Karbach <daniel.karbach@localhorst.tv>
Date: Wed, 29 Aug 2012 20:56:15 +0200
Subject: [PATCH] =?utf8?q?added=20comments=20(/*=E2=80=A6*/=20and=20//?=
 =?utf8?q?=E2=80=A6\n)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=utf8
Content-Transfer-Encoding: 8bit

---
 src/loader/Tokenizer.cpp | 51 +++++++++++++++++++++++++++++++++++++++-
 src/loader/Tokenizer.h   |  8 ++++++-
 2 files changed, 57 insertions(+), 2 deletions(-)

diff --git a/src/loader/Tokenizer.cpp b/src/loader/Tokenizer.cpp
index 995a7e2..6acda5f 100644
--- a/src/loader/Tokenizer.cpp
+++ b/src/loader/Tokenizer.cpp
@@ -12,8 +12,19 @@
 namespace loader {
 
 bool Tokenizer::HasMore() {
+	if (isPutback) return true;
 	ScanSpace();
-	return in;
+	if (!in) return false;
+
+	putback = ReadToken();
+	isPutback = true;
+	if (!skipComments || putback.type != Token::COMMENT) return true;
+
+	while (in && putback.type == Token::COMMENT) {
+		putback = ReadToken();
+		ScanSpace();
+	}
+	return putback.type != Token::COMMENT;
 }
 
 void Tokenizer::ScanSpace() {
@@ -48,6 +59,9 @@ const Tokenizer::Token &Tokenizer::Peek() {
 }
 
 Tokenizer::Token Tokenizer::GetNext() {
+	if (!HasMore()) {
+		throw LexerError(line, "read beyond last token");
+	}
 	if (isPutback) {
 		isPutback = false;
 		return putback;
@@ -80,6 +94,19 @@ Tokenizer::Token Tokenizer::ReadToken() {
 		case '"':
 			in.putback(c);
 			return ReadString();
+		case '/':
+			{
+				std::istream::char_type c2;
+				in.get(c2);
+				if (c2 == '/') {
+					return ReadComment();
+				} else if (c2 == '*') {
+					return ReadMultilineComment();
+				} else {
+					throw LexerError(line, std::string("Tokenizer: cannot parse token: ") + c + c2 + ": expected / or *");
+				}
+			}
+			break;
 		default:
 			in.putback(c);
 			{
@@ -177,6 +204,28 @@ Tokenizer::Token Tokenizer::ReadIdentifier() {
 	return t;
 }
 
+Tokenizer::Token Tokenizer::ReadComment() {
+	std::istream::char_type c;
+	while (in.get(c) && c != '\n');
+	++line;
+	return Token(Token::COMMENT);
+}
+
+Tokenizer::Token Tokenizer::ReadMultilineComment() {
+	std::istream::char_type c;
+	while (in.get(c)) {
+		if (c == '*') {
+			std::istream::char_type c2;
+			if (in.get(c2) && c2 == '/') {
+				break;
+			}
+		} else if (c == '\n') {
+			++line;
+		}
+	}
+	return Token(Token::COMMENT);
+}
+
 bool Tokenizer::CheckKeyword(Token &t) {
 	if (t.str == "export") {
 		t.type = Token::KEYWORD_EXPORT;
diff --git a/src/loader/Tokenizer.h b/src/loader/Tokenizer.h
index 6dda20f..dff96fc 100644
--- a/src/loader/Tokenizer.h
+++ b/src/loader/Tokenizer.h
@@ -18,7 +18,8 @@ namespace loader {
 class Tokenizer {
 
 public:
-	explicit Tokenizer(std::istream &in) : in(in), line(1), isPutback(false) { }
+	explicit Tokenizer(std::istream &in)
+	: in(in), line(1), isPutback(false), skipComments(true) { }
 	~Tokenizer() { }
 private:
 	Tokenizer(const Tokenizer &);
@@ -47,6 +48,7 @@ public:
 			KEYWORD_TRUE = 't',
 			IDENTIFIER = 'x',
 			TYPE_NAME = 'n',
+			COMMENT = 'c'
 		};
 
 		Token() : type(UNKNOWN), number(0) { }
@@ -81,6 +83,9 @@ private:
 	Token ReadString();
 	Token ReadIdentifier();
 
+	Token ReadComment();
+	Token ReadMultilineComment();
+
 	bool CheckKeyword(Token &);
 
 private:
@@ -88,6 +93,7 @@ private:
 	Token putback;
 	int line;
 	bool isPutback;
+	bool skipComments;
 
 };
 
-- 
2.39.2