removed stupid file headers that eclipse put in

[l2e.git] / src / loader / Tokenizer.cpp

diff --git a/src/loader/Tokenizer.cpp b/src/loader/Tokenizer.cpp
index 6acda5fce7ed93dc3df250fd95987b5559211c1f..4a8138c2adab622fb911adae564d1faf58d1210f 100644 (file)
--- a/src/loader/Tokenizer.cpp
+++ b/src/loader/Tokenizer.cpp
@@ -1,10 +1,3 @@
-/*
- * Tokenizer.cpp
- *
- *  Created on: Aug 26, 2012
- *      Author: holy
- */
-
 #include "Tokenizer.h"
 
 #include <istream>
@@ -77,15 +70,35 @@ Tokenizer::Token Tokenizer::ReadToken() {
        switch (c) {
                case Token::ANGLE_BRACKET_OPEN:
                case Token::ANGLE_BRACKET_CLOSE:
-               case Token::CHEVRON_OPEN:
-               case Token::CHEVRON_CLOSE:
                case Token::COLON:
                case Token::COMMA:
                case Token::BRACKET_OPEN:
                case Token::BRACKET_CLOSE:
                case Token::PARENTHESIS_OPEN:
                case Token::PARENTHESIS_CLOSE:
+               case Token::COMMAND:
+               case Token::REGISTER:
                        return Token ((Token::Type) c);
+               case '<': {
+                       std::istream::char_type c2;
+                       in.get(c2);
+                       if (c2 == '<') {
+                               return Token(Token::SCRIPT_BEGIN);
+                       } else {
+                               in.putback(c2);
+                               return Token(Token::CHEVRON_OPEN);
+                       }
+               }
+               case '>': {
+                       std::istream::char_type c2;
+                       in.get(c2);
+                       if (c2 == '>') {
+                               return Token(Token::SCRIPT_END);
+                       } else {
+                               in.putback(c2);
+                               return Token(Token::CHEVRON_CLOSE);
+                       }
+               }
                case '+': case '-':
                case '0': case '1': case '2': case '3': case '4':
                case '5': case '6': case '7': case '8': case '9':