X-Git-Url: http://git.localhorst.tv/?a=blobdiff_plain;f=src%2Floader%2FTokenizer.cpp;h=995a7e2f20469d3ddb1f76231c0cae64c9523058;hb=4b3deb01b66307f5025938538d27e6ad86cfea13;hp=2b87d9c5be43077c186418d75df1cbf041ae93d4;hpb=2aba2da49a94fac57c7d2a469455102cf194957d;p=l2e.git

diff --git a/src/loader/Tokenizer.cpp b/src/loader/Tokenizer.cpp
index 2b87d9c..995a7e2 100644
--- a/src/loader/Tokenizer.cpp
+++ b/src/loader/Tokenizer.cpp
@@ -12,18 +12,41 @@
 namespace loader {
 
 bool Tokenizer::HasMore() {
+	ScanSpace();
 	return in;
 }
 
+void Tokenizer::ScanSpace() {
+	std::istream::char_type c;
+	in.get(c);
+	while (in && std::isspace(c)) {
+		if (c == '\n') {
+			++line;
+		}
+		in.get(c);
+	}
+	if (in) {
+		in.putback(c);
+	}
+}
+
 void Tokenizer::Putback(const Token &t) {
 	if (isPutback) {
-		throw LexerError("Tokenizer: double putback not supported");
+		throw LexerError(line, "Tokenizer: double putback not supported");
 	} else {
 		putback = t;
 		isPutback = true;
 	}
 }
 
+const Tokenizer::Token &Tokenizer::Peek() {
+	if (!isPutback) {
+		putback = GetNext();
+		isPutback = true;
+	}
+	return putback;
+}
+
 Tokenizer::Token Tokenizer::GetNext() {
 	if (isPutback) {
 		isPutback = false;
@@ -34,9 +57,9 @@ Tokenizer::Token Tokenizer::GetNext() {
 }
 
 Tokenizer::Token Tokenizer::ReadToken() {
+	ScanSpace();
 	std::istream::char_type c;
 	in.get(c);
-	while (std::isspace(c)) in.get(c);
 	switch (c) {
 		case Token::ANGLE_BRACKET_OPEN:
 		case Token::ANGLE_BRACKET_CLOSE:
@@ -46,6 +69,8 @@ Tokenizer::Token Tokenizer::ReadToken() {
 		case Token::COMMA:
 		case Token::BRACKET_OPEN:
 		case Token::BRACKET_CLOSE:
+		case Token::PARENTHESIS_OPEN:
+		case Token::PARENTHESIS_CLOSE:
 			return Token ((Token::Type) c);
 		case '+': case '-':
 		case '0': case '1': case '2': case '3': case '4':
@@ -64,7 +89,7 @@ Tokenizer::Token Tokenizer::ReadToken() {
 				} else if (std::islower(c)) {
 					CheckKeyword(t);
 				} else {
-					throw LexerError(std::string("Tokenizer: cannot parse token: ") + c);
+					throw LexerError(line, std::string("Tokenizer: cannot parse token: ") + c);
 				}
 				return t;
 			}
@@ -104,7 +129,7 @@ Tokenizer::Token Tokenizer::ReadString() {
 	std::istream::char_type c;
 	in.get(c);
 	if (c != '"') {
-		throw LexerError("Tokenizer: strings must begin with '\"'");
+		throw LexerError(line, "Tokenizer: strings must begin with '\"'");
 	}
 
 	while (in.get(c)) {