#include "Tokenizer.h"
#include <istream>
-#include <stdexcept>
namespace loader {
bool Tokenizer::HasMore() {
+ ScanSpace();
return in;
}
+void Tokenizer::ScanSpace() {
+ std::istream::char_type c;
+ in.get(c);
+ while (in && std::isspace(c)) {
+ if (c == '\n') {
+ ++line;
+ }
+ in.get(c);
+ }
+ if (in) {
+ in.putback(c);
+ }
+}
+
void Tokenizer::Putback(const Token &t) {
if (isPutback) {
- throw std::runtime_error("Tokenizer: double putback not supported");
+ throw LexerError(line, "Tokenizer: double putback not supported");
} else {
putback = t;
isPutback = true;
}
}
+const Tokenizer::Token &Tokenizer::Peek() {
+ if (!isPutback) {
+ putback = GetNext();
+ isPutback = true;
+ }
+ return putback;
+}
+
Tokenizer::Token Tokenizer::GetNext() {
if (isPutback) {
isPutback = false;
}
Tokenizer::Token Tokenizer::ReadToken() {
+ ScanSpace();
std::istream::char_type c;
in.get(c);
- while (std::isspace(c)) in.get(c);
switch (c) {
case Token::ANGLE_BRACKET_OPEN:
case Token::ANGLE_BRACKET_CLOSE:
case Token::COMMA:
case Token::BRACKET_OPEN:
case Token::BRACKET_CLOSE:
+ case Token::PARENTHESIS_OPEN:
+ case Token::PARENTHESIS_CLOSE:
return Token ((Token::Type) c);
case '+': case '-':
case '0': case '1': case '2': case '3': case '4':
} else if (std::islower(c)) {
CheckKeyword(t);
} else {
- throw std::runtime_error(std::string("Tokenizer: cannot parse token: ") + c);
+ throw LexerError(line, std::string("Tokenizer: cannot parse token: ") + c);
}
return t;
}
std::istream::char_type c;
in.get(c);
if (c != '"') {
- throw std::runtime_error("Tokenizer: strings must begin with '\"'");
+ throw LexerError(line, "Tokenizer: strings must begin with '\"'");
}
while (in.get(c)) {