added new tokens

diff --git a/src/loader/Tokenizer.cpp b/src/loader/Tokenizer.cpp
index 6acda5fce7ed93dc3df250fd95987b5559211c1f..91ad8b9da7ca263db7d84223a8de2cce8228972a 100644 (file)
--- a/src/loader/Tokenizer.cpp
+++ b/src/loader/Tokenizer.cpp
@@ -77,15 +77,35 @@ Tokenizer::Token Tokenizer::ReadToken() {
        switch (c) {
                case Token::ANGLE_BRACKET_OPEN:
                case Token::ANGLE_BRACKET_CLOSE:
-               case Token::CHEVRON_OPEN:
-               case Token::CHEVRON_CLOSE:
                case Token::COLON:
                case Token::COMMA:
                case Token::BRACKET_OPEN:
                case Token::BRACKET_CLOSE:
                case Token::PARENTHESIS_OPEN:
                case Token::PARENTHESIS_CLOSE:
+               case Token::COMMAND:
+               case Token::REGISTER:
                        return Token ((Token::Type) c);
+               case '<': {
+                       std::istream::char_type c2;
+                       in.get(c2);
+                       if (c2 == '<') {
+                               return Token(Token::SCRIPT_BEGIN);
+                       } else {
+                               in.putback(c2);
+                               return Token(Token::CHEVRON_OPEN);
+                       }
+               }
+               case '>': {
+                       std::istream::char_type c2;
+                       in.get(c2);
+                       if (c2 == '>') {
+                               return Token(Token::SCRIPT_END);
+                       } else {
+                               in.putback(c2);
+                               return Token(Token::CHEVRON_CLOSE);
+                       }
+               }
                case '+': case '-':
                case '0': case '1': case '2': case '3': case '4':
                case '5': case '6': case '7': case '8': case '9':

diff --git a/src/loader/Tokenizer.h b/src/loader/Tokenizer.h
index dff96fc6167495769ba82d5a56bd57d6dc9f497a..864f9d2deab832e68627f724b519b5cc9ee8748e 100644 (file)
--- a/src/loader/Tokenizer.h
+++ b/src/loader/Tokenizer.h
@@ -48,7 +48,11 @@ public:
                        KEYWORD_TRUE = 't',
                        IDENTIFIER = 'x',
                        TYPE_NAME = 'n',
-                       COMMENT = 'c'
+                       COMMENT = 'c',
+                       COMMAND = '$',
+                       REGISTER = '%',
+                       SCRIPT_BEGIN = 's',
+                       SCRIPT_END = 'S',
                };
 
                Token() : type(UNKNOWN), number(0) { }
@@ -111,6 +115,8 @@ inline const char *TokenTypeToString(Tokenizer::Token::Type t) {
                        return "COLON";
                case Tokenizer::Token::COMMA:
                        return "COMMA";
+               case Tokenizer::Token::COMMAND:
+                       return "COMMAND";
                case Tokenizer::Token::BRACKET_OPEN:
                        return "BRACKET_OPEN";
                case Tokenizer::Token::BRACKET_CLOSE:
@@ -121,6 +127,12 @@ inline const char *TokenTypeToString(Tokenizer::Token::Type t) {
                        return "PARENTHESIS_CLOSE";
                case Tokenizer::Token::NUMBER:
                        return "NUMBER";
+               case Tokenizer::Token::REGISTER:
+                       return "REGISTER";
+               case Tokenizer::Token::SCRIPT_BEGIN:
+                       return "SCRIPT_BEGIN";
+               case Tokenizer::Token::SCRIPT_END:
+                       return "SCRIPT_END";
                case Tokenizer::Token::STRING:
                        return "STRING";
                case Tokenizer::Token::KEYWORD_EXPORT: