more information in parsed source and output

[l2e.git] / src / loader / Tokenizer.cpp

diff --git a/src/loader/Tokenizer.cpp b/src/loader/Tokenizer.cpp
index cdabe01eed9b8a07c654a2cd068620b953e3f3f9..995a7e2f20469d3ddb1f76231c0cae64c9523058 100644 (file)
--- a/src/loader/Tokenizer.cpp
+++ b/src/loader/Tokenizer.cpp
@@ -12,12 +12,27 @@
 namespace loader {
 
 bool Tokenizer::HasMore() {
-       return std::istream::sentry(in);
+       ScanSpace();
+       return in;
+}
+
+void Tokenizer::ScanSpace() {
+       std::istream::char_type c;
+       in.get(c);
+       while (in && std::isspace(c)) {
+               if (c == '\n') {
+                       ++line;
+               }
+               in.get(c);
+       }
+       if (in) {
+               in.putback(c);
+       }
 }
 
 void Tokenizer::Putback(const Token &t) {
        if (isPutback) {
-               throw LexerError("Tokenizer: double putback not supported");
+               throw LexerError(line, "Tokenizer: double putback not supported");
        } else {
                putback = t;
                isPutback = true;
@@ -42,9 +57,9 @@ Tokenizer::Token Tokenizer::GetNext() {
 }
 
 Tokenizer::Token Tokenizer::ReadToken() {
+       ScanSpace();
        std::istream::char_type c;
        in.get(c);
-       while (std::isspace(c)) in.get(c);
        switch (c) {
                case Token::ANGLE_BRACKET_OPEN:
                case Token::ANGLE_BRACKET_CLOSE:
@@ -74,7 +89,7 @@ Tokenizer::Token Tokenizer::ReadToken() {
                                } else if (std::islower(c)) {
                                        CheckKeyword(t);
                                } else {
-                                       throw LexerError(std::string("Tokenizer: cannot parse token: ") + c);
+                                       throw LexerError(line, std::string("Tokenizer: cannot parse token: ") + c);
                                }
                                return t;
                        }
@@ -114,7 +129,7 @@ Tokenizer::Token Tokenizer::ReadString() {
        std::istream::char_type c;
        in.get(c);
        if (c != '"') {
-               throw LexerError("Tokenizer: strings must begin with '\"'");
+               throw LexerError(line, "Tokenizer: strings must begin with '\"'");
        }
 
        while (in.get(c)) {