1 #include "TokenTest.hpp"
3 #include "io/TokenStreamReader.hpp"
7 #include <glm/gtx/io.hpp>
9 CPPUNIT_TEST_SUITE_REGISTRATION(gong::io::test::TokenTest);
17 void TokenTest::setUp() {
21 void TokenTest::tearDown() {
26 void TokenTest::testTypeIO() {
27 AssertStreamOutput(Token::UNKNOWN, "UNKNOWN");
28 AssertStreamOutput(Token::ANGLE_BRACKET_OPEN, "ANGLE_BRACKET_OPEN");
29 AssertStreamOutput(Token::ANGLE_BRACKET_CLOSE, "ANGLE_BRACKET_CLOSE");
30 AssertStreamOutput(Token::CHEVRON_OPEN, "CHEVRON_OPEN");
31 AssertStreamOutput(Token::CHEVRON_CLOSE, "CHEVRON_CLOSE");
32 AssertStreamOutput(Token::BRACKET_OPEN, "BRACKET_OPEN");
33 AssertStreamOutput(Token::BRACKET_CLOSE, "BRACKET_CLOSE");
34 AssertStreamOutput(Token::PARENTHESIS_OPEN, "PARENTHESIS_OPEN");
35 AssertStreamOutput(Token::PARENTHESIS_CLOSE, "PARENTHESIS_CLOSE");
36 AssertStreamOutput(Token::COLON, "COLON");
37 AssertStreamOutput(Token::SEMICOLON, "SEMICOLON");
38 AssertStreamOutput(Token::COMMA, "COMMA");
39 AssertStreamOutput(Token::EQUALS, "EQUALS");
40 AssertStreamOutput(Token::NUMBER, "NUMBER");
41 AssertStreamOutput(Token::STRING, "STRING");
42 AssertStreamOutput(Token::IDENTIFIER, "IDENTIFIER");
43 AssertStreamOutput(Token::COMMENT, "COMMENT");
46 void TokenTest::testTokenIO() {
48 t.value = "why oh why";
49 AssertStreamOutput(t, "UNKNOWN(why oh why)");
50 t.type = Token::UNKNOWN;
51 t.value = "do I have no purpose";
52 AssertStreamOutput(t, "UNKNOWN(do I have no purpose)");
53 t.type = Token::ANGLE_BRACKET_OPEN;
54 AssertStreamOutput(t, "ANGLE_BRACKET_OPEN");
55 t.type = Token::ANGLE_BRACKET_CLOSE;
56 AssertStreamOutput(t, "ANGLE_BRACKET_CLOSE");
57 t.type = Token::CHEVRON_OPEN;
58 AssertStreamOutput(t, "CHEVRON_OPEN");
59 t.type = Token::CHEVRON_CLOSE;
60 AssertStreamOutput(t, "CHEVRON_CLOSE");
61 t.type = Token::BRACKET_OPEN;
62 AssertStreamOutput(t, "BRACKET_OPEN");
63 t.type = Token::BRACKET_CLOSE;
64 AssertStreamOutput(t, "BRACKET_CLOSE");
65 t.type = Token::PARENTHESIS_OPEN;
66 AssertStreamOutput(t, "PARENTHESIS_OPEN");
67 t.type = Token::PARENTHESIS_CLOSE;
68 AssertStreamOutput(t, "PARENTHESIS_CLOSE");
69 t.type = Token::COLON;
70 AssertStreamOutput(t, "COLON");
71 t.type = Token::SEMICOLON;
72 AssertStreamOutput(t, "SEMICOLON");
73 t.type = Token::COMMA;
74 AssertStreamOutput(t, "COMMA");
75 t.type = Token::EQUALS;
76 AssertStreamOutput(t, "EQUALS");
77 t.type = Token::NUMBER;
79 AssertStreamOutput(t, "NUMBER(15)");
80 t.type = Token::STRING;
81 t.value = "hello world";
82 AssertStreamOutput(t, "STRING(hello world)");
83 t.type = Token::IDENTIFIER;
85 AssertStreamOutput(t, "IDENTIFIER(foo)");
86 t.type = Token::COMMENT;
87 t.value = "WITHOUT ANY WARRANTY";
88 AssertStreamOutput(t, "COMMENT(WITHOUT ANY WARRANTY)");
91 void TokenTest::testTokenizer() {
93 stream << "[{0},<.5>+3=/**\n * test\n */ (-1.5); foo_bar.baz:\"hello\\r\\n\\t\\\"world\\\"\" ] // this line\n#that line";
97 Token token(in.Next());
98 AssertToken(token.type, token.value, in.Current());
99 AssertToken(Token::BRACKET_OPEN, token);
102 AssertToken(Token::ANGLE_BRACKET_OPEN, in.Next());
104 AssertToken(Token::NUMBER, "0", in.Next());
106 AssertToken(Token::ANGLE_BRACKET_CLOSE, in.Next());
108 AssertToken(Token::COMMA, in.Next());
110 AssertToken(Token::CHEVRON_OPEN, in.Next());
112 AssertToken(Token::NUMBER, ".5", in.Next());
114 AssertToken(Token::CHEVRON_CLOSE, in.Next());
116 AssertToken(Token::NUMBER, "+3", in.Next());
118 AssertToken(Token::EQUALS, in.Next());
120 AssertToken(Token::COMMENT, "*\n * test\n ", in.Next());
122 AssertToken(Token::PARENTHESIS_OPEN, in.Next());
124 AssertToken(Token::NUMBER, "-1.5", in.Next());
126 AssertToken(Token::PARENTHESIS_CLOSE, in.Next());
128 AssertToken(Token::SEMICOLON, in.Next());
130 AssertToken(Token::IDENTIFIER, "foo_bar.baz", in.Next());
132 AssertToken(Token::COLON, in.Next());
134 AssertToken(Token::STRING, "hello\r\n\t\"world\"", in.Next());
136 AssertToken(Token::BRACKET_CLOSE, in.Next());
138 AssertToken(Token::COMMENT, " this line", in.Next());
140 AssertToken(Token::COMMENT, "that line", in.Next());
141 CPPUNIT_ASSERT_MESSAGE("expected end of stream", !in.HasMore());
142 CPPUNIT_ASSERT_THROW_MESSAGE(
143 "extracting token after EOS",
144 in.Next(), std::runtime_error);
147 void TokenTest::testTokenizerBrokenComment() {
150 stream << "/* just one more thing…*";
151 Tokenizer in(stream);
153 CPPUNIT_ASSERT_THROW_MESSAGE(
154 "half-closed comment should throw",
155 in.Next(), std::runtime_error);
160 Tokenizer in(stream);
162 CPPUNIT_ASSERT_THROW_MESSAGE(
163 "sole '/' at end of stream should throw",
164 in.Next(), std::runtime_error);
169 Tokenizer in(stream);
171 CPPUNIT_ASSERT_THROW_MESSAGE(
172 "'/' followed by garbage should throw",
173 in.Next(), std::runtime_error);
181 void assert_read(std::string message, T expected, T actual, TokenStreamReader &in) {
183 msg << message << ", current token: " << in.Peek();
184 CPPUNIT_ASSERT_EQUAL_MESSAGE(
191 void TokenTest::testReader() {
195 "true false yes no on off\n"
196 "\"true\" \"false\" \"yes\" \"no\" \"on\" \"off\"\n"
199 "foo foo_bar vec.y\n"
202 ".5 1.5 0.25 -1.75 0.625\n"
205 "\"hello\" \"\" \"\\r\\n\\t\\\"\"\n"
208 "[1,0] [ 0.707, 0.707 ] // vec2\n"
209 "[.577,.577 ,0.577] [ 1,-2,3] // vec3\n"
210 "[ 0, 0, 0, 1 ] [1,0,0,-1.0] // vec4\n"
211 "[640, 480] [3, 4, 5] [0, -10, 100, -1000] # ivecs\n"
212 "[ -0.945, 0, -0.326, 0] # quat\n"
214 TokenStreamReader in(ss);
219 in.ReadBoolean(value_bool);
220 assert_read("reading boolean true", true, value_bool, in);
221 in.ReadBoolean(value_bool);
222 assert_read("reading boolean false", false, value_bool, in);
223 in.ReadBoolean(value_bool);
224 assert_read("reading boolean yes", true, value_bool, in);
225 in.ReadBoolean(value_bool);
226 assert_read("reading boolean no", false, value_bool, in);
227 in.ReadBoolean(value_bool);
228 assert_read("reading boolean on", true, value_bool, in);
229 in.ReadBoolean(value_bool);
230 assert_read("reading boolean off", false, value_bool, in);
232 in.ReadBoolean(value_bool);
233 assert_read("reading boolean \"true\"", true, value_bool, in);
234 in.ReadBoolean(value_bool);
235 assert_read("reading boolean \"false\"", false, value_bool, in);
236 in.ReadBoolean(value_bool);
237 assert_read("reading boolean \"yes\"", true, value_bool, in);
238 in.ReadBoolean(value_bool);
239 assert_read("reading boolean \"no\"", false, value_bool, in);
240 in.ReadBoolean(value_bool);
241 assert_read("reading boolean \"on\"", true, value_bool, in);
242 in.ReadBoolean(value_bool);
243 assert_read("reading boolean \"off\"", false, value_bool, in);
245 in.ReadBoolean(value_bool);
246 assert_read("reading boolean 1", true, value_bool, in);
247 in.ReadBoolean(value_bool);
248 assert_read("reading boolean 0", false, value_bool, in);
249 in.ReadBoolean(value_bool);
250 assert_read("reading boolean -1", true, value_bool, in);
255 in.ReadIdentifier(value_ident);
256 assert_read<string>("reading identifier foo", "foo", value_ident, in);
257 in.ReadIdentifier(value_ident);
258 assert_read<string>("reading identifier foo_bar", "foo_bar", value_ident, in);
259 in.ReadIdentifier(value_ident);
260 assert_read<string>("reading identifier vec.y", "vec.y", value_ident, in);
264 in.ReadNumber(value_int);
265 assert_read("reading integer 0", 0, value_int, in);
266 in.ReadNumber(value_int);
267 assert_read("reading integer 1", 1, value_int, in);
268 in.ReadNumber(value_int);
269 assert_read("reading integer +2", 2, value_int, in);
270 in.ReadNumber(value_int);
271 assert_read("reading integer -3", -3, value_int, in);
272 in.ReadNumber(value_int);
273 assert_read("reading integer 4.5", 4, value_int, in);
276 in.ReadNumber(value_float);
277 assert_read("reading float .5", .5f, value_float, in);
278 in.ReadNumber(value_float);
279 assert_read("reading float 1.5", 1.5f, value_float, in);
280 in.ReadNumber(value_float);
281 assert_read("reading float 0.25", .25f, value_float, in);
282 in.ReadNumber(value_float);
283 assert_read("reading float -1.75", -1.75f, value_float, in);
284 in.ReadNumber(value_float);
285 assert_read("reading float 0.625", 0.625f, value_float, in);
287 unsigned long value_uint;
288 in.ReadNumber(value_uint);
289 assert_read("reading unsigned integer 0", 0ul, value_uint, in);
290 in.ReadNumber(value_uint);
291 assert_read("reading unsigned integer 1", 1ul, value_uint, in);
292 in.ReadNumber(value_uint);
293 assert_read("reading unsigned integer -1", -1ul, value_uint, in);
294 in.ReadNumber(value_uint);
295 assert_read("reading unsigned integer 2.5", 2ul, value_uint, in);
300 in.ReadString(value_string);
302 "reading string \"hello\"",
303 "hello", value_string, in);
304 in.ReadString(value_string);
306 "reading string \"\"",
307 "", value_string, in);
308 in.ReadString(value_string);
310 "reading string \"\\r\\n\\t\\\"\"",
311 "\r\n\t\"", value_string, in);
313 in.ReadRelaxedString(value_string);
315 "reading relaxed string \"world\"",
316 "world", value_string, in);
318 in.ReadRelaxedString(value_string);
320 "reading relaxed string foo",
321 "foo", value_string, in);
323 in.ReadRelaxedString(value_string);
325 "reading relaxed string 12",
326 "12", value_string, in);
330 glm::vec2 value_vec2;
331 in.ReadVec(value_vec2);
333 "reading vector [1,0]",
334 glm::vec2(1, 0), value_vec2, in);
335 in.ReadVec(value_vec2);
337 "reading vector [ 0.707, 0.707 ]",
338 glm::vec2(.707, .707), value_vec2, in);
340 glm::vec3 value_vec3;
341 in.ReadVec(value_vec3);
343 "reading vector [.577,.577 ,0.577]",
344 glm::vec3(.577, .577, .577), value_vec3, in);
345 in.ReadVec(value_vec3);
347 "reading vector [ 1,-2,3]",
348 glm::vec3(1, -2, 3), value_vec3, in);
350 glm::vec4 value_vec4;
351 in.ReadVec(value_vec4);
353 "reading vector [ 0, 0, 0, 1 ]",
354 glm::vec4(0, 0, 0, 1), value_vec4, in);
355 in.ReadVec(value_vec4);
357 "reading vector [1,0,0,-1.0]",
358 glm::vec4(1, 0, 0, -1), value_vec4, in);
360 glm::ivec2 value_ivec2;
361 in.ReadVec(value_ivec2);
363 "reading integer vector [640, 480]",
364 glm::ivec2(640, 480), value_ivec2, in);
365 glm::ivec3 value_ivec3;
366 in.ReadVec(value_ivec3);
368 "reading integer vector [3, 4, 5]",
369 glm::ivec3(3, 4, 5), value_ivec3, in);
370 glm::ivec4 value_ivec4;
371 in.ReadVec(value_ivec4);
373 "reading integer vector [0, -10, 100, -1000]",
374 glm::ivec4(0, -10, 100, -1000), value_ivec4, in);
376 glm::quat value_quat;
377 in.ReadQuat(value_quat);
379 "reading quaternion [ -0.945, 0, -0.326, 0]",
380 glm::quat(-0.945, 0, -0.326, 0), value_quat, in);
381 // TODO: comment at end of stream makes it think there's more?
382 //CPPUNIT_ASSERT_MESSAGE("expected end of stream", !in.HasMore());
383 // TODO: and it even works??
384 //CPPUNIT_ASSERT_THROW_MESSAGE(
385 // "extracting token after EOS",
386 // in.Next(), std::runtime_error);
389 void TokenTest::testReaderEmpty() {
390 { // zero length stream
393 TokenStreamReader in(ss);
394 CPPUNIT_ASSERT_MESSAGE(
395 "empty stream shouldn't have tokens",
398 { // stream consisting solely of comments
405 "// is there anybody out there\n"
407 TokenStreamReader in(ss);
408 CPPUNIT_ASSERT_MESSAGE(
409 "comment stream shouldn't have tokens",
414 void TokenTest::testReaderMalformed() {
418 TokenStreamReader in(ss);
419 CPPUNIT_ASSERT_THROW_MESSAGE(
420 "unexpected token type should throw",
421 in.GetInt(), std::runtime_error);
426 TokenStreamReader in(ss);
427 CPPUNIT_ASSERT_THROW_MESSAGE(
428 "casting ':' to bool should throw",
429 in.GetBool(), std::runtime_error);
434 TokenStreamReader in(ss);
435 CPPUNIT_ASSERT_THROW_MESSAGE(
436 "casting \"hello\" to bool should throw",
437 in.GetBool(), std::runtime_error);
442 void TokenTest::AssertStreamOutput(
448 CPPUNIT_ASSERT_EQUAL_MESSAGE(
449 "unexpected std::ostream << Token::Type result",
450 expected, conv.str());
453 void TokenTest::AssertStreamOutput(
459 CPPUNIT_ASSERT_EQUAL_MESSAGE(
460 "unexpected std::ostream << Token result",
461 expected, conv.str());
464 void TokenTest::AssertHasMore(Tokenizer &in) {
465 CPPUNIT_ASSERT_MESSAGE("unexpected end of stream", in.HasMore());
468 void TokenTest::AssertToken(
469 Token::Type expected_type,
470 const Token &actual_token
472 AssertToken(expected_type, "", actual_token);
475 void TokenTest::AssertToken(
476 Token::Type expected_type,
477 string expected_value,
478 const Token &actual_token
480 CPPUNIT_ASSERT_EQUAL_MESSAGE(
481 "unexpected token type",
482 expected_type, actual_token.type);
483 CPPUNIT_ASSERT_EQUAL_MESSAGE(
484 "unexpected token value",
485 expected_value, actual_token.value);