src/loader/Tokenizer.h

   1 /*
   2  * Tokenizer.h
   3  *
   4  *  Created on: Aug 26, 2012
   5  *      Author: holy
   6  */
   7
   8 #ifndef LOADER_TOKENIZER_H_
   9 #define LOADER_TOKENIZER_H_
  10
  11 #include <iosfwd>
  12 #include <ostream>
  13 #include <string>
  14
  15 namespace loader {
  16
  17 class Tokenizer {
  18
  19 public:
  20         Tokenizer(std::istream &in) : in(in), isPutback(false) { }
  21         ~Tokenizer() { }
  22 private:
  23         Tokenizer(const Tokenizer &);
  24         Tokenizer &operator =(const Tokenizer &);
  25
  26 public:
  27         struct Token {
  28
  29                 enum Type {
  30                         UNKNOWN,
  31                         ANGLE_BRACKET_OPEN = '{',
  32                         ANGLE_BRACKET_CLOSE = '}',
  33                         CHEVRON_OPEN = '<',
  34                         CHEVRON_CLOSE = '>',
  35                         COLON = ':',
  36                         COMMA = ',',
  37                         BRACKET_OPEN = '[',
  38                         BRACKET_CLOSE = ']',
  39                         NUMBER,
  40                         STRING,
  41                         KEYWORD_EXPORT,
  42                         KEYWORD_FALSE,
  43                         KEYWORD_INCLUDE,
  44                         KEYWORD_TRUE,
  45                         IDENTIFIER,
  46                         TYPE_NAME,
  47                 };
  48
  49                 Token() : type(UNKNOWN), number(0) { }
  50                 Token(Type t) : type(t), number(0) { }
  51
  52                 Type type;
  53                 std::string str;
  54                 int number;
  55
  56         };
  57
  58         bool HasMore();
  59         Token GetNext();
  60         void Putback(const Token &);
  61
  62 private:
  63         Token ReadToken();
  64
  65         Token ReadNumber();
  66         Token ReadString();
  67         Token ReadIdentifier();
  68
  69         bool CheckKeyword(Token &);
  70
  71 private:
  72         std::istream &in;
  73         Token putback;
  74         bool isPutback;
  75
  76 };
  77
  78
  79 std::ostream &operator <<(std::ostream &out, Tokenizer::Token::Type t) {
  80         switch (t) {
  81                 case Tokenizer::Token::ANGLE_BRACKET_OPEN:
  82                         out << "ANGLE_BRACKET_OPEN";
  83                         break;
  84                 case Tokenizer::Token::ANGLE_BRACKET_CLOSE:
  85                         out << "ANGLE_BRACKET_CLOSE";
  86                         break;
  87                 case Tokenizer::Token::CHEVRON_OPEN:
  88                         out << "CHEVRON_OPEN";
  89                         break;
  90                 case Tokenizer::Token::CHEVRON_CLOSE:
  91                         out << "CHEVRON_CLOSE";
  92                         break;
  93                 case Tokenizer::Token::COLON:
  94                         out << "COLON";
  95                         break;
  96                 case Tokenizer::Token::COMMA:
  97                         out << "COMMA";
  98                         break;
  99                 case Tokenizer::Token::BRACKET_OPEN:
 100                         out << "BRACKET_OPEN";
 101                         break;
 102                 case Tokenizer::Token::BRACKET_CLOSE:
 103                         out << "BRACKET_CLOSE";
 104                         break;
 105                 case Tokenizer::Token::NUMBER:
 106                         out << "NUMBER";
 107                         break;
 108                 case Tokenizer::Token::STRING:
 109                         out << "STRING";
 110                         break;
 111                 case Tokenizer::Token::KEYWORD_EXPORT:
 112                         out << "KEYWORD_EXPORT";
 113                         break;
 114                 case Tokenizer::Token::KEYWORD_FALSE:
 115                         out << "KEYWORD_FALSE";
 116                         break;
 117                 case Tokenizer::Token::KEYWORD_INCLUDE:
 118                         out << "KEYWORD_INCLUDE";
 119                         break;
 120                 case Tokenizer::Token::KEYWORD_TRUE:
 121                         out << "KEYWORD_TRUE";
 122                         break;
 123                 case Tokenizer::Token::IDENTIFIER:
 124                         out << "IDENTIFIER";
 125                         break;
 126                 case Tokenizer::Token::TYPE_NAME:
 127                         out << "TYPE_NAME";
 128                         break;
 129                 default:
 130                         out << "UNKNOWN";
 131                         break;
 132         }
 133         return out;
 134 }
 135
 136 }
 137
 138 #endif /* LOADER_TOKENIZER_H_ */