test for Tokenizer

diff --git a/tst/io/TokenTest.cpp b/tst/io/TokenTest.cpp
index 52fcfcdf1a5b8b074bffbeebac51daefe3329e5f..062c223a561ab213c6b99679f3502b614a74b9ae 100644 (file)
--- a/tst/io/TokenTest.cpp
+++ b/tst/io/TokenTest.cpp
@@ -1,6 +1,7 @@
 #include "TokenTest.hpp"
 
 #include <sstream>
+#include <stdexcept>
 
 CPPUNIT_TEST_SUITE_REGISTRATION(blank::test::TokenTest);
 
@@ -83,6 +84,89 @@ void TokenTest::testTokenIO() {
        AssertStreamOutput(t, "COMMENT(WITHOUT ANY WARRANTY)");
 }
 
+void TokenTest::testTokenizer() {
+       stringstream stream;
+       stream << "[{0},<.5>+3=/**\n * test\n */ (-1.5); foo_bar.baz:\"hello\\r\\n\\t\\\"world\\\"\" ] // this line\n#that line";
+       Tokenizer in(stream);
+
+       AssertHasMore(in);
+       Token token(in.Next());
+       AssertToken(token.type, token.value, in.Current());
+       AssertToken(Token::BRACKET_OPEN, token);
+
+       AssertHasMore(in);
+       AssertToken(Token::ANGLE_BRACKET_OPEN, in.Next());
+       AssertHasMore(in);
+       AssertToken(Token::NUMBER, "0", in.Next());
+       AssertHasMore(in);
+       AssertToken(Token::ANGLE_BRACKET_CLOSE, in.Next());
+       AssertHasMore(in);
+       AssertToken(Token::COMMA, in.Next());
+       AssertHasMore(in);
+       AssertToken(Token::CHEVRON_OPEN, in.Next());
+       AssertHasMore(in);
+       AssertToken(Token::NUMBER, ".5", in.Next());
+       AssertHasMore(in);
+       AssertToken(Token::CHEVRON_CLOSE, in.Next());
+       AssertHasMore(in);
+       AssertToken(Token::NUMBER, "+3", in.Next());
+       AssertHasMore(in);
+       AssertToken(Token::EQUALS, in.Next());
+       AssertHasMore(in);
+       AssertToken(Token::COMMENT, "*\n * test\n ", in.Next());
+       AssertHasMore(in);
+       AssertToken(Token::PARENTHESIS_OPEN, in.Next());
+       AssertHasMore(in);
+       AssertToken(Token::NUMBER, "-1.5", in.Next());
+       AssertHasMore(in);
+       AssertToken(Token::PARENTHESIS_CLOSE, in.Next());
+       AssertHasMore(in);
+       AssertToken(Token::SEMICOLON, in.Next());
+       AssertHasMore(in);
+       AssertToken(Token::IDENTIFIER, "foo_bar.baz", in.Next());
+       AssertHasMore(in);
+       AssertToken(Token::COLON, in.Next());
+       AssertHasMore(in);
+       AssertToken(Token::STRING, "hello\r\n\t\"world\"", in.Next());
+       AssertHasMore(in);
+       AssertToken(Token::BRACKET_CLOSE, in.Next());
+       AssertHasMore(in);
+       AssertToken(Token::COMMENT, " this line", in.Next());
+       AssertHasMore(in);
+       AssertToken(Token::COMMENT, "that line", in.Next());
+       CPPUNIT_ASSERT_MESSAGE("expected end of stream", !in.HasMore());
+}
+
+void TokenTest::testTokenizerBrokenComment() {
+       {
+               stringstream stream;
+               stream << "/* just one more thing…*";
+               Tokenizer in(stream);
+               AssertHasMore(in);
+               CPPUNIT_ASSERT_THROW_MESSAGE(
+                       "half-closed comment should throw",
+                       in.Next(), std::runtime_error);
+       }
+       {
+               stringstream stream;
+               stream << "  /";
+               Tokenizer in(stream);
+               AssertHasMore(in);
+               CPPUNIT_ASSERT_THROW_MESSAGE(
+                       "sole '/' at end of stream should throw",
+                       in.Next(), std::runtime_error);
+       }
+       {
+               stringstream stream;
+               stream << "/.";
+               Tokenizer in(stream);
+               AssertHasMore(in);
+               CPPUNIT_ASSERT_THROW_MESSAGE(
+                       "'/' followed by garbage should throw",
+                       in.Next(), std::runtime_error);
+       }
+}
+
 
 void TokenTest::AssertStreamOutput(
        Token::Type t,
@@ -106,5 +190,29 @@ void TokenTest::AssertStreamOutput(
                expected, conv.str());
 }
 
+void TokenTest::AssertHasMore(Tokenizer &in) {
+       CPPUNIT_ASSERT_MESSAGE("unexpected end of stream", in.HasMore());
+}
+
+void TokenTest::AssertToken(
+       Token::Type expected_type,
+       const Token &actual_token
+) {
+       AssertToken(expected_type, "", actual_token);
+}
+
+void TokenTest::AssertToken(
+       Token::Type expected_type,
+       string expected_value,
+       const Token &actual_token
+) {
+       CPPUNIT_ASSERT_EQUAL_MESSAGE(
+               "unexpected token type",
+               expected_type, actual_token.type);
+       CPPUNIT_ASSERT_EQUAL_MESSAGE(
+               "unexpected token value",
+               expected_value, actual_token.value);
+}
+
 }
 }

diff --git a/tst/io/TokenTest.hpp b/tst/io/TokenTest.hpp
index 0b78952af623873f83ce75960aad78ef09c6e509..bbe38d4ac41e2fdb1f9c935c8d724bf7f8950107 100644 (file)
--- a/tst/io/TokenTest.hpp
+++ b/tst/io/TokenTest.hpp
@@ -2,6 +2,7 @@
 #define BLANK_TEST_IO_TOKENTEST_HPP
 
 #include "io/Token.hpp"
+#include "io/Tokenizer.hpp"
 
 #include <string>
 #include <cppunit/extensions/HelperMacros.h>
@@ -18,6 +19,8 @@ CPPUNIT_TEST_SUITE(TokenTest);
 
 CPPUNIT_TEST(testTypeIO);
 CPPUNIT_TEST(testTokenIO);
+CPPUNIT_TEST(testTokenizer);
+CPPUNIT_TEST(testTokenizerBrokenComment);
 
 CPPUNIT_TEST_SUITE_END();
 
@@ -27,12 +30,21 @@ public:
 
        void testTypeIO();
        void testTokenIO();
+       void testTokenizer();
+       void testTokenizerBrokenComment();
 
        static void AssertStreamOutput(
                Token::Type, std::string expected);
        static void AssertStreamOutput(
                const Token &, std::string expected);
 
+       static void AssertHasMore(Tokenizer &);
+       static void AssertToken(
+               Token::Type expected_type, const Token &actual_token);
+       static void AssertToken(
+               Token::Type expected_type, std::string expected_value,
+               const Token &actual_token);
+
 };
 
 }