X-Git-Url: http://git.localhorst.tv/?a=blobdiff_plain;f=src%2Floader%2FTokenizer.h;h=864f9d2deab832e68627f724b519b5cc9ee8748e;hb=8fd08e32d902b1340dd686ba0d7990fb1be3b861;hp=6dda20fad0b9744fe6578fdb203b5b50be2000b7;hpb=8d3decb97c252bc7b0ba4b9b5def0b04dfcb8e00;p=l2e.git

diff --git a/src/loader/Tokenizer.h b/src/loader/Tokenizer.h
index 6dda20f..864f9d2 100644
--- a/src/loader/Tokenizer.h
+++ b/src/loader/Tokenizer.h
@@ -18,7 +18,8 @@ namespace loader {
 class Tokenizer {
 
 public:
-	explicit Tokenizer(std::istream &in) : in(in), line(1), isPutback(false) { }
+	explicit Tokenizer(std::istream &in)
+	: in(in), line(1), isPutback(false), skipComments(true) { }
 	~Tokenizer() { }
 private:
 	Tokenizer(const Tokenizer &);
@@ -47,6 +48,11 @@ public:
 			KEYWORD_TRUE = 't',
 			IDENTIFIER = 'x',
 			TYPE_NAME = 'n',
+			COMMENT = 'c',
+			COMMAND = '$',
+			REGISTER = '%',
+			SCRIPT_BEGIN = 's',
+			SCRIPT_END = 'S',
 		};
 
 		Token() : type(UNKNOWN), number(0) { }
@@ -81,6 +87,9 @@ private:
 	Token ReadString();
 	Token ReadIdentifier();
 
+	Token ReadComment();
+	Token ReadMultilineComment();
+
 	bool CheckKeyword(Token &);
 
 private:
@@ -88,6 +97,7 @@ private:
 	Token putback;
 	int line;
 	bool isPutback;
+	bool skipComments;
 
 };
 
@@ -105,6 +115,8 @@ inline const char *TokenTypeToString(Tokenizer::Token::Type t) {
 			return "COLON";
 		case Tokenizer::Token::COMMA:
 			return "COMMA";
+		case Tokenizer::Token::COMMAND:
+			return "COMMAND";
 		case Tokenizer::Token::BRACKET_OPEN:
 			return "BRACKET_OPEN";
 		case Tokenizer::Token::BRACKET_CLOSE:
@@ -115,6 +127,12 @@ inline const char *TokenTypeToString(Tokenizer::Token::Type t) {
 			return "PARENTHESIS_CLOSE";
 		case Tokenizer::Token::NUMBER:
 			return "NUMBER";
+		case Tokenizer::Token::REGISTER:
+			return "REGISTER";
+		case Tokenizer::Token::SCRIPT_BEGIN:
+			return "SCRIPT_BEGIN";
+		case Tokenizer::Token::SCRIPT_END:
+			return "SCRIPT_END";
 		case Tokenizer::Token::STRING:
 			return "STRING";
 		case Tokenizer::Token::KEYWORD_EXPORT: