tst/io/TokenTest.cpp

   1 #include "TokenTest.hpp"
   2
   3 #include "io/TokenStreamReader.hpp"
   4
   5 #include <sstream>
   6 #include <stdexcept>
   7 #include <glm/gtx/io.hpp>
   8
   9 CPPUNIT_TEST_SUITE_REGISTRATION(blank::test::TokenTest);
  10
  11 using namespace std;
  12
  13 namespace blank {
  14 namespace test {
  15
  16 void TokenTest::setUp() {
  17
  18 }
  19
  20 void TokenTest::tearDown() {
  21
  22 }
  23
  24
  25 void TokenTest::testTypeIO() {
  26         AssertStreamOutput(Token::UNKNOWN, "UNKNOWN");
  27         AssertStreamOutput(Token::ANGLE_BRACKET_OPEN, "ANGLE_BRACKET_OPEN");
  28         AssertStreamOutput(Token::ANGLE_BRACKET_CLOSE, "ANGLE_BRACKET_CLOSE");
  29         AssertStreamOutput(Token::CHEVRON_OPEN, "CHEVRON_OPEN");
  30         AssertStreamOutput(Token::CHEVRON_CLOSE, "CHEVRON_CLOSE");
  31         AssertStreamOutput(Token::BRACKET_OPEN, "BRACKET_OPEN");
  32         AssertStreamOutput(Token::BRACKET_CLOSE, "BRACKET_CLOSE");
  33         AssertStreamOutput(Token::PARENTHESIS_OPEN, "PARENTHESIS_OPEN");
  34         AssertStreamOutput(Token::PARENTHESIS_CLOSE, "PARENTHESIS_CLOSE");
  35         AssertStreamOutput(Token::COLON, "COLON");
  36         AssertStreamOutput(Token::SEMICOLON, "SEMICOLON");
  37         AssertStreamOutput(Token::COMMA, "COMMA");
  38         AssertStreamOutput(Token::EQUALS, "EQUALS");
  39         AssertStreamOutput(Token::NUMBER, "NUMBER");
  40         AssertStreamOutput(Token::STRING, "STRING");
  41         AssertStreamOutput(Token::IDENTIFIER, "IDENTIFIER");
  42         AssertStreamOutput(Token::COMMENT, "COMMENT");
  43 }
  44
  45 void TokenTest::testTokenIO() {
  46         Token t;
  47         t.value = "why oh why";
  48         AssertStreamOutput(t, "UNKNOWN(why oh why)");
  49         t.type = Token::UNKNOWN;
  50         t.value = "do I have no purpose";
  51         AssertStreamOutput(t, "UNKNOWN(do I have no purpose)");
  52         t.type = Token::ANGLE_BRACKET_OPEN;
  53         AssertStreamOutput(t, "ANGLE_BRACKET_OPEN");
  54         t.type = Token::ANGLE_BRACKET_CLOSE;
  55         AssertStreamOutput(t, "ANGLE_BRACKET_CLOSE");
  56         t.type = Token::CHEVRON_OPEN;
  57         AssertStreamOutput(t, "CHEVRON_OPEN");
  58         t.type = Token::CHEVRON_CLOSE;
  59         AssertStreamOutput(t, "CHEVRON_CLOSE");
  60         t.type = Token::BRACKET_OPEN;
  61         AssertStreamOutput(t, "BRACKET_OPEN");
  62         t.type = Token::BRACKET_CLOSE;
  63         AssertStreamOutput(t, "BRACKET_CLOSE");
  64         t.type = Token::PARENTHESIS_OPEN;
  65         AssertStreamOutput(t, "PARENTHESIS_OPEN");
  66         t.type = Token::PARENTHESIS_CLOSE;
  67         AssertStreamOutput(t, "PARENTHESIS_CLOSE");
  68         t.type = Token::COLON;
  69         AssertStreamOutput(t, "COLON");
  70         t.type = Token::SEMICOLON;
  71         AssertStreamOutput(t, "SEMICOLON");
  72         t.type = Token::COMMA;
  73         AssertStreamOutput(t, "COMMA");
  74         t.type = Token::EQUALS;
  75         AssertStreamOutput(t, "EQUALS");
  76         t.type = Token::NUMBER;
  77         t.value = "15";
  78         AssertStreamOutput(t, "NUMBER(15)");
  79         t.type = Token::STRING;
  80         t.value = "hello world";
  81         AssertStreamOutput(t, "STRING(hello world)");
  82         t.type = Token::IDENTIFIER;
  83         t.value = "foo";
  84         AssertStreamOutput(t, "IDENTIFIER(foo)");
  85         t.type = Token::COMMENT;
  86         t.value = "WITHOUT ANY WARRANTY";
  87         AssertStreamOutput(t, "COMMENT(WITHOUT ANY WARRANTY)");
  88 }
  89
  90 void TokenTest::testTokenizer() {
  91         stringstream stream;
  92         stream << "[{0},<.5>+3=/**\n * test\n */ (-1.5); foo_bar.baz:\"hello\\r\\n\\t\\\"world\\\"\" ] // this line\n#that line";
  93         Tokenizer in(stream);
  94
  95         AssertHasMore(in);
  96         Token token(in.Next());
  97         AssertToken(token.type, token.value, in.Current());
  98         AssertToken(Token::BRACKET_OPEN, token);
  99
 100         AssertHasMore(in);
 101         AssertToken(Token::ANGLE_BRACKET_OPEN, in.Next());
 102         AssertHasMore(in);
 103         AssertToken(Token::NUMBER, "0", in.Next());
 104         AssertHasMore(in);
 105         AssertToken(Token::ANGLE_BRACKET_CLOSE, in.Next());
 106         AssertHasMore(in);
 107         AssertToken(Token::COMMA, in.Next());
 108         AssertHasMore(in);
 109         AssertToken(Token::CHEVRON_OPEN, in.Next());
 110         AssertHasMore(in);
 111         AssertToken(Token::NUMBER, ".5", in.Next());
 112         AssertHasMore(in);
 113         AssertToken(Token::CHEVRON_CLOSE, in.Next());
 114         AssertHasMore(in);
 115         AssertToken(Token::NUMBER, "+3", in.Next());
 116         AssertHasMore(in);
 117         AssertToken(Token::EQUALS, in.Next());
 118         AssertHasMore(in);
 119         AssertToken(Token::COMMENT, "*\n * test\n ", in.Next());
 120         AssertHasMore(in);
 121         AssertToken(Token::PARENTHESIS_OPEN, in.Next());
 122         AssertHasMore(in);
 123         AssertToken(Token::NUMBER, "-1.5", in.Next());
 124         AssertHasMore(in);
 125         AssertToken(Token::PARENTHESIS_CLOSE, in.Next());
 126         AssertHasMore(in);
 127         AssertToken(Token::SEMICOLON, in.Next());
 128         AssertHasMore(in);
 129         AssertToken(Token::IDENTIFIER, "foo_bar.baz", in.Next());
 130         AssertHasMore(in);
 131         AssertToken(Token::COLON, in.Next());
 132         AssertHasMore(in);
 133         AssertToken(Token::STRING, "hello\r\n\t\"world\"", in.Next());
 134         AssertHasMore(in);
 135         AssertToken(Token::BRACKET_CLOSE, in.Next());
 136         AssertHasMore(in);
 137         AssertToken(Token::COMMENT, " this line", in.Next());
 138         AssertHasMore(in);
 139         AssertToken(Token::COMMENT, "that line", in.Next());
 140         CPPUNIT_ASSERT_MESSAGE("expected end of stream", !in.HasMore());
 141         CPPUNIT_ASSERT_THROW_MESSAGE(
 142                 "extracting token after EOS",
 143                 in.Next(), std::runtime_error);
 144 }
 145
 146 void TokenTest::testTokenizerBrokenComment() {
 147         {
 148                 stringstream stream;
 149                 stream << "/* just one more thing…*";
 150                 Tokenizer in(stream);
 151                 AssertHasMore(in);
 152                 CPPUNIT_ASSERT_THROW_MESSAGE(
 153                         "half-closed comment should throw",
 154                         in.Next(), std::runtime_error);
 155         }
 156         {
 157                 stringstream stream;
 158                 stream << "  /";
 159                 Tokenizer in(stream);
 160                 AssertHasMore(in);
 161                 CPPUNIT_ASSERT_THROW_MESSAGE(
 162                         "sole '/' at end of stream should throw",
 163                         in.Next(), std::runtime_error);
 164         }
 165         {
 166                 stringstream stream;
 167                 stream << "/.";
 168                 Tokenizer in(stream);
 169                 AssertHasMore(in);
 170                 CPPUNIT_ASSERT_THROW_MESSAGE(
 171                         "'/' followed by garbage should throw",
 172                         in.Next(), std::runtime_error);
 173         }
 174 }
 175
 176
 177 namespace {
 178
 179 template<class T>
 180 void assert_read(std::string message, T expected, T actual, TokenStreamReader &in) {
 181         stringstream msg;
 182         msg << message << ", current token: " << in.Peek();
 183         CPPUNIT_ASSERT_EQUAL_MESSAGE(
 184                 msg.str(),
 185                 expected, actual);
 186 }
 187
 188 }
 189
 190 void TokenTest::testReader() {
 191         stringstream ss;
 192         ss <<
 193                 "/* booleans */\n"
 194                 "true false yes no on off\n"
 195                 "\"true\" \"false\" \"yes\" \"no\" \"on\" \"off\"\n"
 196                 "1 0 -1\n"
 197                 "# identifiers\n"
 198                 "foo foo_bar vec.y\n"
 199                 "// numbers\n"
 200                 "0 1 +2 -3 4.5\n"
 201                 ".5 1.5 0.25 -1.75 0.625\n"
 202                 "0 1 -1 2.5\n"
 203                 // strings
 204                 "\"hello\" \"\" \"\\r\\n\\t\\\"\"\n"
 205                 "\"world\" foo 12\n"
 206                 // vectors
 207                 "[1,0] [ 0.707, 0.707 ] // vec2\n"
 208                 "[.577,.577 ,0.577] [ 1,-2,3] // vec3\n"
 209                 "[ 0, 0, 0, 1 ] [1,0,0,-1.0] // vec4\n"
 210                 "[640, 480] [3, 4, 5] [0, -10, 100, -1000] # ivecs\n"
 211                 "[ -0.945, 0, -0.326, 0] # quat\n"
 212                 ;
 213         TokenStreamReader in(ss);
 214
 215         // booleans
 216
 217         bool value_bool;
 218         in.ReadBoolean(value_bool);
 219         assert_read("reading boolean true", true, value_bool, in);
 220         in.ReadBoolean(value_bool);
 221         assert_read("reading boolean false", false, value_bool, in);
 222         in.ReadBoolean(value_bool);
 223         assert_read("reading boolean yes", true, value_bool, in);
 224         in.ReadBoolean(value_bool);
 225         assert_read("reading boolean no", false, value_bool, in);
 226         in.ReadBoolean(value_bool);
 227         assert_read("reading boolean on", true, value_bool, in);
 228         in.ReadBoolean(value_bool);
 229         assert_read("reading boolean off", false, value_bool, in);
 230
 231         in.ReadBoolean(value_bool);
 232         assert_read("reading boolean \"true\"", true, value_bool, in);
 233         in.ReadBoolean(value_bool);
 234         assert_read("reading boolean \"false\"", false, value_bool, in);
 235         in.ReadBoolean(value_bool);
 236         assert_read("reading boolean \"yes\"", true, value_bool, in);
 237         in.ReadBoolean(value_bool);
 238         assert_read("reading boolean \"no\"", false, value_bool, in);
 239         in.ReadBoolean(value_bool);
 240         assert_read("reading boolean \"on\"", true, value_bool, in);
 241         in.ReadBoolean(value_bool);
 242         assert_read("reading boolean \"off\"", false, value_bool, in);
 243
 244         in.ReadBoolean(value_bool);
 245         assert_read("reading boolean 1", true, value_bool, in);
 246         in.ReadBoolean(value_bool);
 247         assert_read("reading boolean 0", false, value_bool, in);
 248         in.ReadBoolean(value_bool);
 249         assert_read("reading boolean -1", true, value_bool, in);
 250
 251         // identifiers
 252
 253         string value_ident;
 254         in.ReadIdentifier(value_ident);
 255         assert_read<string>("reading identifier foo", "foo", value_ident, in);
 256         in.ReadIdentifier(value_ident);
 257         assert_read<string>("reading identifier foo_bar", "foo_bar", value_ident, in);
 258         in.ReadIdentifier(value_ident);
 259         assert_read<string>("reading identifier vec.y", "vec.y", value_ident, in);
 260
 261         // numbers
 262         int value_int;
 263         in.ReadNumber(value_int);
 264         assert_read("reading integer 0", 0, value_int, in);
 265         in.ReadNumber(value_int);
 266         assert_read("reading integer 1", 1, value_int, in);
 267         in.ReadNumber(value_int);
 268         assert_read("reading integer +2", 2, value_int, in);
 269         in.ReadNumber(value_int);
 270         assert_read("reading integer -3", -3, value_int, in);
 271         in.ReadNumber(value_int);
 272         assert_read("reading integer 4.5", 4, value_int, in);
 273
 274         float value_float;
 275         in.ReadNumber(value_float);
 276         assert_read("reading float .5", .5f, value_float, in);
 277         in.ReadNumber(value_float);
 278         assert_read("reading float 1.5", 1.5f, value_float, in);
 279         in.ReadNumber(value_float);
 280         assert_read("reading float 0.25", .25f, value_float, in);
 281         in.ReadNumber(value_float);
 282         assert_read("reading float -1.75", -1.75f, value_float, in);
 283         in.ReadNumber(value_float);
 284         assert_read("reading float 0.625", 0.625f, value_float, in);
 285
 286         unsigned long value_uint;
 287         in.ReadNumber(value_uint);
 288         assert_read("reading unsigned integer 0", 0ul, value_uint, in);
 289         in.ReadNumber(value_uint);
 290         assert_read("reading unsigned integer 1", 1ul, value_uint, in);
 291         in.ReadNumber(value_uint);
 292         assert_read("reading unsigned integer -1", -1ul, value_uint, in);
 293         in.ReadNumber(value_uint);
 294         assert_read("reading unsigned integer 2.5", 2ul, value_uint, in);
 295
 296         // strings
 297
 298         string value_string;
 299         in.ReadString(value_string);
 300         assert_read<string>(
 301                 "reading string \"hello\"",
 302                 "hello", value_string, in);
 303         in.ReadString(value_string);
 304         assert_read<string>(
 305                 "reading string \"\"",
 306                 "", value_string, in);
 307         in.ReadString(value_string);
 308         assert_read<string>(
 309                 "reading string \"\\r\\n\\t\\\"\"",
 310                 "\r\n\t\"", value_string, in);
 311
 312         in.ReadRelaxedString(value_string);
 313         assert_read<string>(
 314                 "reading relaxed string \"world\"",
 315                 "world", value_string, in);
 316
 317         in.ReadRelaxedString(value_string);
 318         assert_read<string>(
 319                 "reading relaxed string foo",
 320                 "foo", value_string, in);
 321
 322         in.ReadRelaxedString(value_string);
 323         assert_read<string>(
 324                 "reading relaxed string 12",
 325                 "12", value_string, in);
 326
 327         // vectors
 328
 329         glm::vec2 value_vec2;
 330         in.ReadVec(value_vec2);
 331         assert_read(
 332                 "reading vector [1,0]",
 333                 glm::vec2(1, 0), value_vec2, in);
 334         in.ReadVec(value_vec2);
 335         assert_read(
 336                 "reading vector [ 0.707, 0.707 ]",
 337                 glm::vec2(.707, .707), value_vec2, in);
 338
 339         glm::vec3 value_vec3;
 340         in.ReadVec(value_vec3);
 341         assert_read(
 342                 "reading vector [.577,.577 ,0.577]",
 343                 glm::vec3(.577, .577, .577), value_vec3, in);
 344         in.ReadVec(value_vec3);
 345         assert_read(
 346                 "reading vector [ 1,-2,3]",
 347                 glm::vec3(1, -2, 3), value_vec3, in);
 348
 349         glm::vec4 value_vec4;
 350         in.ReadVec(value_vec4);
 351         assert_read(
 352                 "reading vector [ 0, 0, 0, 1 ]",
 353                 glm::vec4(0, 0, 0, 1), value_vec4, in);
 354         in.ReadVec(value_vec4);
 355         assert_read(
 356                 "reading vector [1,0,0,-1.0]",
 357                 glm::vec4(1, 0, 0, -1), value_vec4, in);
 358
 359         glm::ivec2 value_ivec2;
 360         in.ReadVec(value_ivec2);
 361         assert_read(
 362                 "reading integer vector [640, 480]",
 363                 glm::ivec2(640, 480), value_ivec2, in);
 364         glm::ivec3 value_ivec3;
 365         in.ReadVec(value_ivec3);
 366         assert_read(
 367                 "reading integer vector [3, 4, 5]",
 368                 glm::ivec3(3, 4, 5), value_ivec3, in);
 369         glm::ivec4 value_ivec4;
 370         in.ReadVec(value_ivec4);
 371         assert_read(
 372                 "reading integer vector [0, -10, 100, -1000]",
 373                 glm::ivec4(0, -10, 100, -1000), value_ivec4, in);
 374
 375         glm::quat value_quat;
 376         in.ReadQuat(value_quat);
 377         assert_read(
 378                 "reading quaternion [ -0.945, 0, -0.326, 0]",
 379                 glm::quat(-0.945, 0, -0.326, 0), value_quat, in);
 380         // TODO: comment at end of stream makes it think there's more?
 381         //CPPUNIT_ASSERT_MESSAGE("expected end of stream", !in.HasMore());
 382         // TODO: and it even works??
 383         //CPPUNIT_ASSERT_THROW_MESSAGE(
 384         //      "extracting token after EOS",
 385         //      in.Next(), std::runtime_error);
 386 }
 387
 388 void TokenTest::testReaderEmpty() {
 389         { // zero length stream
 390                 stringstream ss;
 391                 ss << "";
 392                 TokenStreamReader in(ss);
 393                 CPPUNIT_ASSERT_MESSAGE(
 394                         "empty stream shouldn't have tokens",
 395                         !in.HasMore());
 396         }
 397         { // stream consisting solely of comments
 398                 stringstream ss;
 399                 ss <<
 400                         "/*\n"
 401                         " * hello\n"
 402                         " */\n"
 403                         "#hello\n"
 404                         "// is there anybody out there\n"
 405                         ;
 406                 TokenStreamReader in(ss);
 407                 CPPUNIT_ASSERT_MESSAGE(
 408                         "comment stream shouldn't have tokens",
 409                         !in.HasMore());
 410         }
 411 }
 412
 413 void TokenTest::testReaderMalformed() {
 414         {
 415                 stringstream ss;
 416                 ss << "a";
 417                 TokenStreamReader in(ss);
 418                 CPPUNIT_ASSERT_THROW_MESSAGE(
 419                         "unexpected token type should throw",
 420                         in.GetInt(), std::runtime_error);
 421         }
 422         {
 423                 stringstream ss;
 424                 ss << ":";
 425                 TokenStreamReader in(ss);
 426                 CPPUNIT_ASSERT_THROW_MESSAGE(
 427                         "casting ':' to bool should throw",
 428                         in.GetBool(), std::runtime_error);
 429         }
 430         {
 431                 stringstream ss;
 432                 ss << "hello";
 433                 TokenStreamReader in(ss);
 434                 CPPUNIT_ASSERT_THROW_MESSAGE(
 435                         "casting \"hello\" to bool should throw",
 436                         in.GetBool(), std::runtime_error);
 437         }
 438 }
 439
 440
 441 void TokenTest::AssertStreamOutput(
 442         Token::Type t,
 443         string expected
 444 ) {
 445         stringstream conv;
 446         conv << t;
 447         CPPUNIT_ASSERT_EQUAL_MESSAGE(
 448                 "unexpected std::ostream << Token::Type result",
 449                 expected, conv.str());
 450 }
 451
 452 void TokenTest::AssertStreamOutput(
 453         const Token &t,
 454         string expected
 455 ) {
 456         stringstream conv;
 457         conv << t;
 458         CPPUNIT_ASSERT_EQUAL_MESSAGE(
 459                 "unexpected std::ostream << Token result",
 460                 expected, conv.str());
 461 }
 462
 463 void TokenTest::AssertHasMore(Tokenizer &in) {
 464         CPPUNIT_ASSERT_MESSAGE("unexpected end of stream", in.HasMore());
 465 }
 466
 467 void TokenTest::AssertToken(
 468         Token::Type expected_type,
 469         const Token &actual_token
 470 ) {
 471         AssertToken(expected_type, "", actual_token);
 472 }
 473
 474 void TokenTest::AssertToken(
 475         Token::Type expected_type,
 476         string expected_value,
 477         const Token &actual_token
 478 ) {
 479         CPPUNIT_ASSERT_EQUAL_MESSAGE(
 480                 "unexpected token type",
 481                 expected_type, actual_token.type);
 482         CPPUNIT_ASSERT_EQUAL_MESSAGE(
 483                 "unexpected token value",
 484                 expected_value, actual_token.value);
 485 }
 486
 487 }
 488 }