added peek function to tokenizer

diff --git a/src/loader/Tokenizer.cpp b/src/loader/Tokenizer.cpp
index 828f6ed131afe44118e8b86aead5b2c2235284ea..c5cc561b004f128ed94fd984dfa8d45afa18ccc1 100644 (file)
--- a/src/loader/Tokenizer.cpp
+++ b/src/loader/Tokenizer.cpp
@@ -24,6 +24,14 @@ void Tokenizer::Putback(const Token &t) {
        }
 }
 
+const Tokenizer::Token &Tokenizer::Peek() {
+       if (!isPutback) {
+               putback = GetNext();
+               isPutback = true;
+       }
+       return putback;
+}
+
 Tokenizer::Token Tokenizer::GetNext() {
        if (isPutback) {
                isPutback = false;

diff --git a/src/loader/Tokenizer.h b/src/loader/Tokenizer.h
index 80a31693beb7e21abb19d03340964b7e08a67314..b7ca72f664c148c1f92029a27cc132fbdb7682ce 100644 (file)
--- a/src/loader/Tokenizer.h
+++ b/src/loader/Tokenizer.h
@@ -28,7 +28,7 @@ public:
        struct Token {
 
                enum Type {
-                       UNKNOWN,
+                       UNKNOWN = 0,
                        ANGLE_BRACKET_OPEN = '{',
                        ANGLE_BRACKET_CLOSE = '}',
                        CHEVRON_OPEN = '<',
@@ -39,14 +39,14 @@ public:
                        BRACKET_CLOSE = ']',
                        PARENTHESIS_OPEN = '(',
                        PARENTHESIS_CLOSE = ')',
-                       NUMBER,
-                       STRING,
-                       KEYWORD_EXPORT,
-                       KEYWORD_FALSE,
-                       KEYWORD_INCLUDE,
-                       KEYWORD_TRUE,
-                       IDENTIFIER,
-                       TYPE_NAME,
+                       NUMBER = '0',
+                       STRING = '"',
+                       KEYWORD_EXPORT = 'e',
+                       KEYWORD_FALSE = 'f',
+                       KEYWORD_INCLUDE = 'i',
+                       KEYWORD_TRUE = 't',
+                       IDENTIFIER = 'x',
+                       TYPE_NAME = 'n',
                };
 
                Token() : type(UNKNOWN), number(0) { }
@@ -65,6 +65,7 @@ public:
 
        bool HasMore();
        Token GetNext();
+       const Token &Peek();
        void Putback(const Token &);
 
 private: