4 * Created on: Aug 26, 2012
14 bool Tokenizer::HasMore() {
18 void Tokenizer::Putback(const Token &t) {
20 throw LexerError("Tokenizer: double putback not supported");
27 Tokenizer::Token Tokenizer::GetNext() {
36 Tokenizer::Token Tokenizer::ReadToken() {
37 std::istream::char_type c;
39 while (std::isspace(c)) in.get(c);
41 case Token::ANGLE_BRACKET_OPEN:
42 case Token::ANGLE_BRACKET_CLOSE:
43 case Token::CHEVRON_OPEN:
44 case Token::CHEVRON_CLOSE:
47 case Token::BRACKET_OPEN:
48 case Token::BRACKET_CLOSE:
49 return Token ((Token::Type) c);
51 case '0': case '1': case '2': case '3': case '4':
52 case '5': case '6': case '7': case '8': case '9':
61 Token t(ReadIdentifier());
62 if (std::isupper(c)) {
63 t.type = Token::TYPE_NAME;
64 } else if (std::islower(c)) {
67 throw LexerError(std::string("Tokenizer: cannot parse token: ") + c);
74 Tokenizer::Token Tokenizer::ReadNumber() {
75 Token t(Token::NUMBER);
76 bool isNegative(false);
78 std::istream::char_type c;
82 } else if (c != '+') {
87 if (!std::isdigit(c)) {
95 if (isNegative) t.number *= -1;
100 Tokenizer::Token Tokenizer::ReadString() {
101 Token t(Token::STRING);
104 std::istream::char_type c;
107 throw LexerError("Tokenizer: strings must begin with '\"'");
115 t.str.push_back('\n');
118 t.str.push_back('\r');
121 t.str.push_back('\t');
127 } else if (c == '"') {
129 } else if (c == '\\') {
139 Tokenizer::Token Tokenizer::ReadIdentifier() {
140 Token t(Token::IDENTIFIER);
142 std::istream::char_type c;
144 if (std::isalnum(c) || c == '_') {
155 bool Tokenizer::CheckKeyword(Token &t) {
156 if (t.str == "export") {
157 t.type = Token::KEYWORD_EXPORT;
159 } else if (t.str == "false") {
160 t.type = Token::KEYWORD_FALSE;
162 } else if (t.str == "include") {
163 t.type = Token::KEYWORD_INCLUDE;
165 } else if (t.str == "true") {
166 t.type = Token::KEYWORD_TRUE;