class Tokenizer {
public:
- explicit Tokenizer(std::istream &in) : in(in), line(1), isPutback(false) { }
+ explicit Tokenizer(std::istream &in)
+ : in(in), line(1), isPutback(false), skipComments(true) { }
~Tokenizer() { }
private:
Tokenizer(const Tokenizer &);
KEYWORD_TRUE = 't',
IDENTIFIER = 'x',
TYPE_NAME = 'n',
+ COMMENT = 'c',
+ COMMAND = '$',
+ REGISTER = '%',
+ SCRIPT_BEGIN = 's',
+ SCRIPT_END = 'S',
};
Token() : type(UNKNOWN), number(0) { }
Token ReadString();
Token ReadIdentifier();
+ Token ReadComment();
+ Token ReadMultilineComment();
+
bool CheckKeyword(Token &);
private:
Token putback;
int line;
bool isPutback;
+ bool skipComments;
};
return "COLON";
case Tokenizer::Token::COMMA:
return "COMMA";
+ case Tokenizer::Token::COMMAND:
+ return "COMMAND";
case Tokenizer::Token::BRACKET_OPEN:
return "BRACKET_OPEN";
case Tokenizer::Token::BRACKET_CLOSE:
return "PARENTHESIS_CLOSE";
case Tokenizer::Token::NUMBER:
return "NUMBER";
+ case Tokenizer::Token::REGISTER:
+ return "REGISTER";
+ case Tokenizer::Token::SCRIPT_BEGIN:
+ return "SCRIPT_BEGIN";
+ case Tokenizer::Token::SCRIPT_END:
+ return "SCRIPT_END";
case Tokenizer::Token::STRING:
return "STRING";
case Tokenizer::Token::KEYWORD_EXPORT: