X-Git-Url: http://git.localhorst.tv/?a=blobdiff_plain;f=src%2Fio%2Ftoken.cpp;h=76a9c2f355e42a54f26f758b3bd76466160fc5aa;hb=c6ca9d21e45af5ea7caeec722a9b59fdf3aa3b24;hp=5e734ad573b634ed4fcf101c89735f7c64dda4b2;hpb=ede25c0a2f59e21521d1cd962e6ea9d78169ca12;p=blank.git

diff --git a/src/io/token.cpp b/src/io/token.cpp
index 5e734ad..76a9c2f 100644
--- a/src/io/token.cpp
+++ b/src/io/token.cpp
@@ -32,7 +32,7 @@ void Tokenizer::ReadToken() {
 
 	istream::sentry s(in);
 	if (!s) {
-		// TODO: error?
+		throw runtime_error("read past the end of stream");
 		return;
 	}
 
@@ -47,7 +47,7 @@ void Tokenizer::ReadToken() {
 		case ',': case '=':
 			current.type = Token::Type(c);
 			break;
-		case '+': case '-':
+		case '+': case '-': case '.':
 		case '0': case '1': case '2': case '3': case '4':
 		case '5': case '6': case '7': case '8': case '9':
 			in.putback(c);
@@ -71,7 +71,7 @@ void Tokenizer::ReadToken() {
 namespace {
 
 bool is_num_char(istream::char_type c) {
-	return isdigit(c)
+	return isxdigit(c)
 		|| c == '.'
 		|| c == '-'
 		|| c == '+'
@@ -191,6 +191,9 @@ TokenStreamReader::TokenStreamReader(istream &in)
 
 
 bool TokenStreamReader::HasMore() {
+	if (cached) {
+		return true;
+	}
 	while (in.HasMore()) {
 		if (in.Next().type != Token::COMMENT) {
 			cached = true;