+/*
+ * Tokenizer.cpp
+ *
+ * Created on: Aug 26, 2012
+ * Author: holy
+ */
+
+#include "Tokenizer.h"
+
+#include <istream>
+#include <stdexcept>
+
+namespace loader {
+
+bool Tokenizer::HasMore() {
+ return in;
+}
+
+void Tokenizer::Putback(const Token &t) {
+ if (isPutback) {
+ throw std::runtime_error("Tokenizer: double putback not supported");
+ } else {
+ putback = t;
+ isPutback = true;
+ }
+}
+
+Tokenizer::Token Tokenizer::GetNext() {
+ if (isPutback) {
+ isPutback = false;
+ return putback;
+ } else {
+ return ReadToken();
+ }
+}
+
+Tokenizer::Token Tokenizer::ReadToken() {
+ std::istream::char_type c;
+ in.get(c);
+ while (std::isspace(c)) in.get(c);
+ switch (c) {
+ case Token::ANGLE_BRACKET_OPEN:
+ case Token::ANGLE_BRACKET_CLOSE:
+ case Token::CHEVRON_OPEN:
+ case Token::CHEVRON_CLOSE:
+ case Token::COLON:
+ case Token::COMMA:
+ case Token::BRACKET_OPEN:
+ case Token::BRACKET_CLOSE:
+ return (Token::Type) c;
+ case '+': case '-':
+ case '0': case '1': case '2': case '3': case '4':
+ case '5': case '6': case '7': case '8': case '9':
+ in.putback(c);
+ return ReadNumber();
+ case '"':
+ in.putback(c);
+ return ReadString();
+ default:
+ in.putback(c);
+ {
+ Token t(ReadIdentifier());
+ if (std::isupper(c)) {
+ t.type = Token::TYPE_NAME;
+ } else if (std::islower(c)) {
+ CheckKeyword(t);
+ } else {
+ throw std::runtime_error(std::string("Tokenizer: cannot parse token: ") + c);
+ }
+ return t;
+ }
+ }
+}
+
+Tokenizer::Token Tokenizer::ReadNumber() {
+ Token t(Token::NUMBER);
+ bool isNegative(false);
+
+ std::istream::char_type c;
+ in.get(c);
+ if (c == '-') {
+ isNegative = true;
+ } else if (c != '+') {
+ in.putback(c);
+ }
+
+ while (in.get(c)) {
+ if (!std::isdigit(c)) {
+ in.putback(c);
+ break;
+ }
+ t.number *= 10;
+ t.number += c - '0';
+ }
+
+ if (isNegative) t.number *= -1;
+
+ return t;
+}
+
+Tokenizer::Token Tokenizer::ReadString() {
+ Token t(Token::STRING);
+ bool escape(false);
+
+ std::istream::char_type c;
+ in.get(c);
+ if (c != '"') {
+ throw std::runtime_error("Tokenizer: strings must begin with '\"'");
+ }
+
+ while (in.get(c)) {
+ if (escape) {
+ escape = false;
+ switch (c) {
+ case 'n':
+ t.str.push_back('\n');
+ break;
+ case 'r':
+ t.str.push_back('\r');
+ break;
+ case 't':
+ t.str.push_back('\t');
+ break;
+ default:
+ t.str.push_back(c);
+ break;
+ }
+ } else if (c == '"') {
+ break;
+ } else if (c == '\\') {
+ escape = true;
+ } else {
+ t.str.push_back(c);
+ }
+ }
+
+ return t;
+}
+
+Tokenizer::Token Tokenizer::ReadIdentifier() {
+ Token t(Token::IDENTIFIER);
+
+ std::istream::char_type c;
+ while (in.get(c)) {
+ if (std::isalnum(c) || c == '_') {
+ t.str.push_back(c);
+ } else {
+ in.putback(c);
+ break;
+ }
+ }
+
+ return t;
+}
+
+bool Tokenizer::CheckKeyword(Token &t) {
+ if (t.str == "export") {
+ t.type = Token::KEYWORD_EXPORT;
+ return true;
+ } else if (t.str == "false") {
+ t.type = Token::KEYWORD_FALSE;
+ return true;
+ } else if (t.str == "include") {
+ t.type = Token::KEYWORD_INCLUDE;
+ return true;
+ } else if (t.str == "true") {
+ t.type = Token::KEYWORD_TRUE;
+ return true;
+ } else {
+ return false;
+ }
+}
+
+}