#include <iosfwd>
#include <ostream>
+#include <stdexcept>
#include <string>
namespace loader {
class Tokenizer {
public:
- Tokenizer(std::istream &in) : in(in), isPutback(false) { }
+ explicit Tokenizer(std::istream &in)
+ : in(in), line(1), isPutback(false), skipComments(true) { }
~Tokenizer() { }
private:
Tokenizer(const Tokenizer &);
struct Token {
enum Type {
- UNKNOWN,
+ UNKNOWN = 0,
ANGLE_BRACKET_OPEN = '{',
ANGLE_BRACKET_CLOSE = '}',
CHEVRON_OPEN = '<',
COMMA = ',',
BRACKET_OPEN = '[',
BRACKET_CLOSE = ']',
- NUMBER,
- STRING,
- KEYWORD_EXPORT,
- KEYWORD_FALSE,
- KEYWORD_INCLUDE,
- KEYWORD_TRUE,
- IDENTIFIER,
- TYPE_NAME,
+ PARENTHESIS_OPEN = '(',
+ PARENTHESIS_CLOSE = ')',
+ NUMBER = '0',
+ STRING = '"',
+ KEYWORD_EXPORT = 'e',
+ KEYWORD_FALSE = 'f',
+ KEYWORD_INCLUDE = 'i',
+ KEYWORD_TRUE = 't',
+ IDENTIFIER = 'x',
+ TYPE_NAME = 'n',
+ COMMENT = 'c'
};
Token() : type(UNKNOWN), number(0) { }
- Token(Type t) : type(t), number(0) { }
+ explicit Token(Type t) : type(t), number(0) { }
Type type;
std::string str;
};
+ class LexerError: public std::runtime_error {
+ public:
+ LexerError(int line, const std::string &msg)
+ : std::runtime_error(msg), line(line) { }
+ int Line() const { return line; }
+ private:
+ int line;
+ };
+
bool HasMore();
Token GetNext();
+ const Token &Peek();
void Putback(const Token &);
+ int Line() const { return line; }
private:
+ void ScanSpace();
Token ReadToken();
Token ReadNumber();
Token ReadString();
Token ReadIdentifier();
+ Token ReadComment();
+ Token ReadMultilineComment();
+
bool CheckKeyword(Token &);
private:
std::istream ∈
Token putback;
+ int line;
bool isPutback;
+ bool skipComments;
};
-
-std::ostream &operator <<(std::ostream &out, Tokenizer::Token::Type t) {
+inline const char *TokenTypeToString(Tokenizer::Token::Type t) {
switch (t) {
case Tokenizer::Token::ANGLE_BRACKET_OPEN:
- out << "ANGLE_BRACKET_OPEN";
- break;
+ return "ANGLE_BRACKET_OPEN";
case Tokenizer::Token::ANGLE_BRACKET_CLOSE:
- out << "ANGLE_BRACKET_CLOSE";
- break;
+ return "ANGLE_BRACKET_CLOSE";
case Tokenizer::Token::CHEVRON_OPEN:
- out << "CHEVRON_OPEN";
- break;
+ return "CHEVRON_OPEN";
case Tokenizer::Token::CHEVRON_CLOSE:
- out << "CHEVRON_CLOSE";
- break;
+ return "CHEVRON_CLOSE";
case Tokenizer::Token::COLON:
- out << "COLON";
- break;
+ return "COLON";
case Tokenizer::Token::COMMA:
- out << "COMMA";
- break;
+ return "COMMA";
case Tokenizer::Token::BRACKET_OPEN:
- out << "BRACKET_OPEN";
- break;
+ return "BRACKET_OPEN";
case Tokenizer::Token::BRACKET_CLOSE:
- out << "BRACKET_CLOSE";
- break;
+ return "BRACKET_CLOSE";
+ case Tokenizer::Token::PARENTHESIS_OPEN:
+ return "PARENTHESIS_OPEN";
+ case Tokenizer::Token::PARENTHESIS_CLOSE:
+ return "PARENTHESIS_CLOSE";
case Tokenizer::Token::NUMBER:
- out << "NUMBER";
- break;
+ return "NUMBER";
case Tokenizer::Token::STRING:
- out << "STRING";
- break;
+ return "STRING";
case Tokenizer::Token::KEYWORD_EXPORT:
- out << "KEYWORD_EXPORT";
- break;
+ return "KEYWORD_EXPORT";
case Tokenizer::Token::KEYWORD_FALSE:
- out << "KEYWORD_FALSE";
- break;
+ return "KEYWORD_FALSE";
case Tokenizer::Token::KEYWORD_INCLUDE:
- out << "KEYWORD_INCLUDE";
- break;
+ return "KEYWORD_INCLUDE";
case Tokenizer::Token::KEYWORD_TRUE:
- out << "KEYWORD_TRUE";
- break;
+ return "KEYWORD_TRUE";
case Tokenizer::Token::IDENTIFIER:
- out << "IDENTIFIER";
- break;
+ return "IDENTIFIER";
case Tokenizer::Token::TYPE_NAME:
- out << "TYPE_NAME";
- break;
+ return "TYPE_NAME";
default:
- out << "UNKNOWN";
- break;
+ return "UNKNOWN";
}
+}
+
+inline std::ostream &operator <<(std::ostream &out, Tokenizer::Token::Type t) {
+ out << TokenTypeToString(t);
return out;
}