X-Git-Url: http://git.localhorst.tv/?a=blobdiff_plain;f=src%2Floader%2FTokenizer.cpp;h=4a8138c2adab622fb911adae564d1faf58d1210f;hb=HEAD;hp=6acda5fce7ed93dc3df250fd95987b5559211c1f;hpb=51e0a94d0e0a3bc1a4664aa9af1f20910f55201c;p=l2e.git

diff --git a/src/loader/Tokenizer.cpp b/src/loader/Tokenizer.cpp
index 6acda5f..4a8138c 100644
--- a/src/loader/Tokenizer.cpp
+++ b/src/loader/Tokenizer.cpp
@@ -1,10 +1,3 @@
-/*
- * Tokenizer.cpp
- *
- *  Created on: Aug 26, 2012
- *      Author: holy
- */
-
 #include "Tokenizer.h"
 
 #include <istream>
@@ -77,15 +70,35 @@ Tokenizer::Token Tokenizer::ReadToken() {
 	switch (c) {
 		case Token::ANGLE_BRACKET_OPEN:
 		case Token::ANGLE_BRACKET_CLOSE:
-		case Token::CHEVRON_OPEN:
-		case Token::CHEVRON_CLOSE:
 		case Token::COLON:
 		case Token::COMMA:
 		case Token::BRACKET_OPEN:
 		case Token::BRACKET_CLOSE:
 		case Token::PARENTHESIS_OPEN:
 		case Token::PARENTHESIS_CLOSE:
+		case Token::COMMAND:
+		case Token::REGISTER:
 			return Token ((Token::Type) c);
+		case '<': {
+			std::istream::char_type c2;
+			in.get(c2);
+			if (c2 == '<') {
+				return Token(Token::SCRIPT_BEGIN);
+			} else {
+				in.putback(c2);
+				return Token(Token::CHEVRON_OPEN);
+			}
+		}
+		case '>': {
+			std::istream::char_type c2;
+			in.get(c2);
+			if (c2 == '>') {
+				return Token(Token::SCRIPT_END);
+			} else {
+				in.putback(c2);
+				return Token(Token::CHEVRON_CLOSE);
+			}
+		}
 		case '+': case '-':
 		case '0': case '1': case '2': case '3': case '4':
 		case '5': case '6': case '7': case '8': case '9':