removed stupid file headers that eclipse put in

[l2e.git] / src / loader / Tokenizer.h

diff --git a/src/loader/Tokenizer.h b/src/loader/Tokenizer.h
index 6dda20fad0b9744fe6578fdb203b5b50be2000b7..06103b7d8b6f41378a32839405f1d58df4964755 100644 (file)
--- a/src/loader/Tokenizer.h
+++ b/src/loader/Tokenizer.h
@@ -1,10 +1,3 @@
-/*
- * Tokenizer.h
- *
- *  Created on: Aug 26, 2012
- *      Author: holy
- */
-
 #ifndef LOADER_TOKENIZER_H_
 #define LOADER_TOKENIZER_H_
 
@@ -18,7 +11,8 @@ namespace loader {
 class Tokenizer {
 
 public:
-       explicit Tokenizer(std::istream &in) : in(in), line(1), isPutback(false) { }
+       explicit Tokenizer(std::istream &in)
+       : in(in), line(1), isPutback(false), skipComments(true) { }
        ~Tokenizer() { }
 private:
        Tokenizer(const Tokenizer &);
@@ -47,6 +41,11 @@ public:
                        KEYWORD_TRUE = 't',
                        IDENTIFIER = 'x',
                        TYPE_NAME = 'n',
+                       COMMENT = 'c',
+                       COMMAND = '$',
+                       REGISTER = '%',
+                       SCRIPT_BEGIN = 's',
+                       SCRIPT_END = 'S',
                };
 
                Token() : type(UNKNOWN), number(0) { }
@@ -81,6 +80,9 @@ private:
        Token ReadString();
        Token ReadIdentifier();
 
+       Token ReadComment();
+       Token ReadMultilineComment();
+
        bool CheckKeyword(Token &);
 
 private:
@@ -88,6 +90,7 @@ private:
        Token putback;
        int line;
        bool isPutback;
+       bool skipComments;
 
 };
 
@@ -105,6 +108,8 @@ inline const char *TokenTypeToString(Tokenizer::Token::Type t) {
                        return "COLON";
                case Tokenizer::Token::COMMA:
                        return "COMMA";
+               case Tokenizer::Token::COMMAND:
+                       return "COMMAND";
                case Tokenizer::Token::BRACKET_OPEN:
                        return "BRACKET_OPEN";
                case Tokenizer::Token::BRACKET_CLOSE:
@@ -115,6 +120,12 @@ inline const char *TokenTypeToString(Tokenizer::Token::Type t) {
                        return "PARENTHESIS_CLOSE";
                case Tokenizer::Token::NUMBER:
                        return "NUMBER";
+               case Tokenizer::Token::REGISTER:
+                       return "REGISTER";
+               case Tokenizer::Token::SCRIPT_BEGIN:
+                       return "SCRIPT_BEGIN";
+               case Tokenizer::Token::SCRIPT_END:
+                       return "SCRIPT_END";
                case Tokenizer::Token::STRING:
                        return "STRING";
                case Tokenizer::Token::KEYWORD_EXPORT: