src/loader/Tokenizer.h

   1 /*
   2  * Tokenizer.h
   3  *
   4  *  Created on: Aug 26, 2012
   5  *      Author: holy
   6  */
   7
   8 #ifndef LOADER_TOKENIZER_H_
   9 #define LOADER_TOKENIZER_H_
  10
  11 #include <iosfwd>
  12 #include <ostream>
  13 #include <stdexcept>
  14 #include <string>
  15
  16 namespace loader {
  17
  18 class Tokenizer {
  19
  20 public:
  21         explicit Tokenizer(std::istream &in) : in(in), isPutback(false) { }
  22         ~Tokenizer() { }
  23 private:
  24         Tokenizer(const Tokenizer &);
  25         Tokenizer &operator =(const Tokenizer &);
  26
  27 public:
  28         struct Token {
  29
  30                 enum Type {
  31                         UNKNOWN = 0,
  32                         ANGLE_BRACKET_OPEN = '{',
  33                         ANGLE_BRACKET_CLOSE = '}',
  34                         CHEVRON_OPEN = '<',
  35                         CHEVRON_CLOSE = '>',
  36                         COLON = ':',
  37                         COMMA = ',',
  38                         BRACKET_OPEN = '[',
  39                         BRACKET_CLOSE = ']',
  40                         PARENTHESIS_OPEN = '(',
  41                         PARENTHESIS_CLOSE = ')',
  42                         NUMBER = '0',
  43                         STRING = '"',
  44                         KEYWORD_EXPORT = 'e',
  45                         KEYWORD_FALSE = 'f',
  46                         KEYWORD_INCLUDE = 'i',
  47                         KEYWORD_TRUE = 't',
  48                         IDENTIFIER = 'x',
  49                         TYPE_NAME = 'n',
  50                 };
  51
  52                 Token() : type(UNKNOWN), number(0) { }
  53                 explicit Token(Type t) : type(t), number(0) { }
  54
  55                 Type type;
  56                 std::string str;
  57                 int number;
  58
  59         };
  60
  61         class LexerError: public std::runtime_error {
  62         public:
  63                 explicit LexerError(const std::string &msg) : std::runtime_error(msg) { }
  64         };
  65
  66         bool HasMore();
  67         Token GetNext();
  68         const Token &Peek();
  69         void Putback(const Token &);
  70
  71 private:
  72         Token ReadToken();
  73
  74         Token ReadNumber();
  75         Token ReadString();
  76         Token ReadIdentifier();
  77
  78         bool CheckKeyword(Token &);
  79
  80 private:
  81         std::istream &in;
  82         Token putback;
  83         bool isPutback;
  84
  85 };
  86
  87 inline const char *TokenTypeToString(Tokenizer::Token::Type t) {
  88         switch (t) {
  89                 case Tokenizer::Token::ANGLE_BRACKET_OPEN:
  90                         return "ANGLE_BRACKET_OPEN";
  91                 case Tokenizer::Token::ANGLE_BRACKET_CLOSE:
  92                         return "ANGLE_BRACKET_CLOSE";
  93                 case Tokenizer::Token::CHEVRON_OPEN:
  94                         return "CHEVRON_OPEN";
  95                 case Tokenizer::Token::CHEVRON_CLOSE:
  96                         return "CHEVRON_CLOSE";
  97                 case Tokenizer::Token::COLON:
  98                         return "COLON";
  99                 case Tokenizer::Token::COMMA:
 100                         return "COMMA";
 101                 case Tokenizer::Token::BRACKET_OPEN:
 102                         return "BRACKET_OPEN";
 103                 case Tokenizer::Token::BRACKET_CLOSE:
 104                         return "BRACKET_CLOSE";
 105                 case Tokenizer::Token::PARENTHESIS_OPEN:
 106                         return "PARENTHESIS_OPEN";
 107                 case Tokenizer::Token::PARENTHESIS_CLOSE:
 108                         return "PARENTHESIS_CLOSE";
 109                 case Tokenizer::Token::NUMBER:
 110                         return "NUMBER";
 111                 case Tokenizer::Token::STRING:
 112                         return "STRING";
 113                 case Tokenizer::Token::KEYWORD_EXPORT:
 114                         return "KEYWORD_EXPORT";
 115                 case Tokenizer::Token::KEYWORD_FALSE:
 116                         return "KEYWORD_FALSE";
 117                 case Tokenizer::Token::KEYWORD_INCLUDE:
 118                         return "KEYWORD_INCLUDE";
 119                 case Tokenizer::Token::KEYWORD_TRUE:
 120                         return "KEYWORD_TRUE";
 121                 case Tokenizer::Token::IDENTIFIER:
 122                         return "IDENTIFIER";
 123                 case Tokenizer::Token::TYPE_NAME:
 124                         return "TYPE_NAME";
 125                 default:
 126                         return "UNKNOWN";
 127         }
 128 }
 129
 130 inline std::ostream &operator <<(std::ostream &out, Tokenizer::Token::Type t) {
 131         out << TokenTypeToString(t);
 132         return out;
 133 }
 134
 135 }
 136
 137 #endif /* LOADER_TOKENIZER_H_ */