X-Git-Url: http://git.localhorst.tv/?a=blobdiff_plain;f=src%2Floader%2FTokenizer.h;h=dff96fc6167495769ba82d5a56bd57d6dc9f497a;hb=51e0a94d0e0a3bc1a4664aa9af1f20910f55201c;hp=80a31693beb7e21abb19d03340964b7e08a67314;hpb=d1bfdd8fcb19f27d63400109810de168d0a8ab1a;p=l2e.git

diff --git a/src/loader/Tokenizer.h b/src/loader/Tokenizer.h
index 80a3169..dff96fc 100644
--- a/src/loader/Tokenizer.h
+++ b/src/loader/Tokenizer.h
@@ -18,7 +18,8 @@ namespace loader {
 class Tokenizer {
 
 public:
-	explicit Tokenizer(std::istream &in) : in(in), isPutback(false) { }
+	explicit Tokenizer(std::istream &in)
+	: in(in), line(1), isPutback(false), skipComments(true) { }
 	~Tokenizer() { }
 private:
 	Tokenizer(const Tokenizer &);
@@ -28,7 +29,7 @@ public:
 	struct Token {
 
 		enum Type {
-			UNKNOWN,
+			UNKNOWN = 0,
 			ANGLE_BRACKET_OPEN = '{',
 			ANGLE_BRACKET_CLOSE = '}',
 			CHEVRON_OPEN = '<',
@@ -39,14 +40,15 @@ public:
 			BRACKET_CLOSE = ']',
 			PARENTHESIS_OPEN = '(',
 			PARENTHESIS_CLOSE = ')',
-			NUMBER,
-			STRING,
-			KEYWORD_EXPORT,
-			KEYWORD_FALSE,
-			KEYWORD_INCLUDE,
-			KEYWORD_TRUE,
-			IDENTIFIER,
-			TYPE_NAME,
+			NUMBER = '0',
+			STRING = '"',
+			KEYWORD_EXPORT = 'e',
+			KEYWORD_FALSE = 'f',
+			KEYWORD_INCLUDE = 'i',
+			KEYWORD_TRUE = 't',
+			IDENTIFIER = 'x',
+			TYPE_NAME = 'n',
+			COMMENT = 'c'
 		};
 
 		Token() : type(UNKNOWN), number(0) { }
@@ -60,26 +62,38 @@ public:
 
 	class LexerError: public std::runtime_error {
 	public:
-		explicit LexerError(const std::string &msg) : std::runtime_error(msg) { }
+		LexerError(int line, const std::string &msg)
+		: std::runtime_error(msg), line(line) { }
+		int Line() const { return line; }
+	private:
+		int line;
 	};
 
 	bool HasMore();
 	Token GetNext();
+	const Token &Peek();
 	void Putback(const Token &);
+	int Line() const { return line; }
 
 private:
+	void ScanSpace();
 	Token ReadToken();
 
 	Token ReadNumber();
 	Token ReadString();
 	Token ReadIdentifier();
 
+	Token ReadComment();
+	Token ReadMultilineComment();
+
 	bool CheckKeyword(Token &);
 
 private:
 	std::istream &in;
 	Token putback;
+	int line;
 	bool isPutback;
+	bool skipComments;
 
 };