namespace loader {
bool Tokenizer::HasMore() {
+ if (isPutback) return true;
ScanSpace();
- return in;
+ if (!in) return false;
+
+ putback = ReadToken();
+ isPutback = true;
+ if (!skipComments || putback.type != Token::COMMENT) return true;
+
+ while (in && putback.type == Token::COMMENT) {
+ putback = ReadToken();
+ ScanSpace();
+ }
+ return putback.type != Token::COMMENT;
}
void Tokenizer::ScanSpace() {
}
Tokenizer::Token Tokenizer::GetNext() {
+ if (!HasMore()) {
+ throw LexerError(line, "read beyond last token");
+ }
if (isPutback) {
isPutback = false;
return putback;
switch (c) {
case Token::ANGLE_BRACKET_OPEN:
case Token::ANGLE_BRACKET_CLOSE:
- case Token::CHEVRON_OPEN:
- case Token::CHEVRON_CLOSE:
case Token::COLON:
case Token::COMMA:
case Token::BRACKET_OPEN:
case Token::BRACKET_CLOSE:
case Token::PARENTHESIS_OPEN:
case Token::PARENTHESIS_CLOSE:
+ case Token::COMMAND:
+ case Token::REGISTER:
return Token ((Token::Type) c);
+ case '<': {
+ std::istream::char_type c2;
+ in.get(c2);
+ if (c2 == '<') {
+ return Token(Token::SCRIPT_BEGIN);
+ } else {
+ in.putback(c2);
+ return Token(Token::CHEVRON_OPEN);
+ }
+ }
+ case '>': {
+ std::istream::char_type c2;
+ in.get(c2);
+ if (c2 == '>') {
+ return Token(Token::SCRIPT_END);
+ } else {
+ in.putback(c2);
+ return Token(Token::CHEVRON_CLOSE);
+ }
+ }
case '+': case '-':
case '0': case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9':
case '"':
in.putback(c);
return ReadString();
+ case '/':
+ {
+ std::istream::char_type c2;
+ in.get(c2);
+ if (c2 == '/') {
+ return ReadComment();
+ } else if (c2 == '*') {
+ return ReadMultilineComment();
+ } else {
+ throw LexerError(line, std::string("Tokenizer: cannot parse token: ") + c + c2 + ": expected / or *");
+ }
+ }
+ break;
default:
in.putback(c);
{
return t;
}
+Tokenizer::Token Tokenizer::ReadComment() {
+ std::istream::char_type c;
+ while (in.get(c) && c != '\n');
+ ++line;
+ return Token(Token::COMMENT);
+}
+
+Tokenizer::Token Tokenizer::ReadMultilineComment() {
+ std::istream::char_type c;
+ while (in.get(c)) {
+ if (c == '*') {
+ std::istream::char_type c2;
+ if (in.get(c2) && c2 == '/') {
+ break;
+ }
+ } else if (c == '\n') {
+ ++line;
+ }
+ }
+ return Token(Token::COMMENT);
+}
+
bool Tokenizer::CheckKeyword(Token &t) {
if (t.str == "export") {
t.type = Token::KEYWORD_EXPORT;