From: Daniel Karbach <daniel.karbach@localhorst.tv>
Date: Sun, 14 Oct 2012 12:44:38 +0000 (+0200)
Subject: added new tokens
X-Git-Url: http://git.localhorst.tv/?a=commitdiff_plain;h=963ed8ddb873e0638b55b38c5590db75db8254c8;p=l2e.git

added new tokens
---

diff --git a/src/loader/Tokenizer.cpp b/src/loader/Tokenizer.cpp
index 6acda5f..91ad8b9 100644
--- a/src/loader/Tokenizer.cpp
+++ b/src/loader/Tokenizer.cpp
@@ -77,15 +77,35 @@ Tokenizer::Token Tokenizer::ReadToken() {
 	switch (c) {
 		case Token::ANGLE_BRACKET_OPEN:
 		case Token::ANGLE_BRACKET_CLOSE:
-		case Token::CHEVRON_OPEN:
-		case Token::CHEVRON_CLOSE:
 		case Token::COLON:
 		case Token::COMMA:
 		case Token::BRACKET_OPEN:
 		case Token::BRACKET_CLOSE:
 		case Token::PARENTHESIS_OPEN:
 		case Token::PARENTHESIS_CLOSE:
+		case Token::COMMAND:
+		case Token::REGISTER:
 			return Token ((Token::Type) c);
+		case '<': {
+			std::istream::char_type c2;
+			in.get(c2);
+			if (c2 == '<') {
+				return Token(Token::SCRIPT_BEGIN);
+			} else {
+				in.putback(c2);
+				return Token(Token::CHEVRON_OPEN);
+			}
+		}
+		case '>': {
+			std::istream::char_type c2;
+			in.get(c2);
+			if (c2 == '>') {
+				return Token(Token::SCRIPT_END);
+			} else {
+				in.putback(c2);
+				return Token(Token::CHEVRON_CLOSE);
+			}
+		}
 		case '+': case '-':
 		case '0': case '1': case '2': case '3': case '4':
 		case '5': case '6': case '7': case '8': case '9':
diff --git a/src/loader/Tokenizer.h b/src/loader/Tokenizer.h
index dff96fc..864f9d2 100644
--- a/src/loader/Tokenizer.h
+++ b/src/loader/Tokenizer.h
@@ -48,7 +48,11 @@ public:
 			KEYWORD_TRUE = 't',
 			IDENTIFIER = 'x',
 			TYPE_NAME = 'n',
-			COMMENT = 'c'
+			COMMENT = 'c',
+			COMMAND = '$',
+			REGISTER = '%',
+			SCRIPT_BEGIN = 's',
+			SCRIPT_END = 'S',
 		};
 
 		Token() : type(UNKNOWN), number(0) { }
@@ -111,6 +115,8 @@ inline const char *TokenTypeToString(Tokenizer::Token::Type t) {
 			return "COLON";
 		case Tokenizer::Token::COMMA:
 			return "COMMA";
+		case Tokenizer::Token::COMMAND:
+			return "COMMAND";
 		case Tokenizer::Token::BRACKET_OPEN:
 			return "BRACKET_OPEN";
 		case Tokenizer::Token::BRACKET_CLOSE:
@@ -121,6 +127,12 @@ inline const char *TokenTypeToString(Tokenizer::Token::Type t) {
 			return "PARENTHESIS_CLOSE";
 		case Tokenizer::Token::NUMBER:
 			return "NUMBER";
+		case Tokenizer::Token::REGISTER:
+			return "REGISTER";
+		case Tokenizer::Token::SCRIPT_BEGIN:
+			return "SCRIPT_BEGIN";
+		case Tokenizer::Token::SCRIPT_END:
+			return "SCRIPT_END";
 		case Tokenizer::Token::STRING:
 			return "STRING";
 		case Tokenizer::Token::KEYWORD_EXPORT: