tst/io/TokenTest.cpp

   1 #include "TokenTest.hpp"
   2
   3 #include "io/TokenStreamReader.hpp"
   4
   5 #include <sstream>
   6 #include <stdexcept>
   7 #include <glm/gtx/io.hpp>
   8
   9 CPPUNIT_TEST_SUITE_REGISTRATION(gong::io::test::TokenTest);
  10
  11 using namespace std;
  12
  13 namespace gong {
  14 namespace io {
  15 namespace test {
  16
  17 void TokenTest::setUp() {
  18
  19 }
  20
  21 void TokenTest::tearDown() {
  22
  23 }
  24
  25
  26 void TokenTest::testTypeIO() {
  27         AssertStreamOutput(Token::UNKNOWN, "UNKNOWN");
  28         AssertStreamOutput(Token::ANGLE_BRACKET_OPEN, "ANGLE_BRACKET_OPEN");
  29         AssertStreamOutput(Token::ANGLE_BRACKET_CLOSE, "ANGLE_BRACKET_CLOSE");
  30         AssertStreamOutput(Token::CHEVRON_OPEN, "CHEVRON_OPEN");
  31         AssertStreamOutput(Token::CHEVRON_CLOSE, "CHEVRON_CLOSE");
  32         AssertStreamOutput(Token::BRACKET_OPEN, "BRACKET_OPEN");
  33         AssertStreamOutput(Token::BRACKET_CLOSE, "BRACKET_CLOSE");
  34         AssertStreamOutput(Token::PARENTHESIS_OPEN, "PARENTHESIS_OPEN");
  35         AssertStreamOutput(Token::PARENTHESIS_CLOSE, "PARENTHESIS_CLOSE");
  36         AssertStreamOutput(Token::COLON, "COLON");
  37         AssertStreamOutput(Token::SEMICOLON, "SEMICOLON");
  38         AssertStreamOutput(Token::COMMA, "COMMA");
  39         AssertStreamOutput(Token::EQUALS, "EQUALS");
  40         AssertStreamOutput(Token::NUMBER, "NUMBER");
  41         AssertStreamOutput(Token::STRING, "STRING");
  42         AssertStreamOutput(Token::IDENTIFIER, "IDENTIFIER");
  43         AssertStreamOutput(Token::COMMENT, "COMMENT");
  44 }
  45
  46 void TokenTest::testTokenIO() {
  47         Token t;
  48         t.value = "why oh why";
  49         AssertStreamOutput(t, "UNKNOWN(why oh why)");
  50         t.type = Token::UNKNOWN;
  51         t.value = "do I have no purpose";
  52         AssertStreamOutput(t, "UNKNOWN(do I have no purpose)");
  53         t.type = Token::ANGLE_BRACKET_OPEN;
  54         AssertStreamOutput(t, "ANGLE_BRACKET_OPEN");
  55         t.type = Token::ANGLE_BRACKET_CLOSE;
  56         AssertStreamOutput(t, "ANGLE_BRACKET_CLOSE");
  57         t.type = Token::CHEVRON_OPEN;
  58         AssertStreamOutput(t, "CHEVRON_OPEN");
  59         t.type = Token::CHEVRON_CLOSE;
  60         AssertStreamOutput(t, "CHEVRON_CLOSE");
  61         t.type = Token::BRACKET_OPEN;
  62         AssertStreamOutput(t, "BRACKET_OPEN");
  63         t.type = Token::BRACKET_CLOSE;
  64         AssertStreamOutput(t, "BRACKET_CLOSE");
  65         t.type = Token::PARENTHESIS_OPEN;
  66         AssertStreamOutput(t, "PARENTHESIS_OPEN");
  67         t.type = Token::PARENTHESIS_CLOSE;
  68         AssertStreamOutput(t, "PARENTHESIS_CLOSE");
  69         t.type = Token::COLON;
  70         AssertStreamOutput(t, "COLON");
  71         t.type = Token::SEMICOLON;
  72         AssertStreamOutput(t, "SEMICOLON");
  73         t.type = Token::COMMA;
  74         AssertStreamOutput(t, "COMMA");
  75         t.type = Token::EQUALS;
  76         AssertStreamOutput(t, "EQUALS");
  77         t.type = Token::NUMBER;
  78         t.value = "15";
  79         AssertStreamOutput(t, "NUMBER(15)");
  80         t.type = Token::STRING;
  81         t.value = "hello world";
  82         AssertStreamOutput(t, "STRING(hello world)");
  83         t.type = Token::IDENTIFIER;
  84         t.value = "foo";
  85         AssertStreamOutput(t, "IDENTIFIER(foo)");
  86         t.type = Token::COMMENT;
  87         t.value = "WITHOUT ANY WARRANTY";
  88         AssertStreamOutput(t, "COMMENT(WITHOUT ANY WARRANTY)");
  89 }
  90
  91 void TokenTest::testTokenizer() {
  92         stringstream stream;
  93         stream << "[{0},<.5>+3=/**\n * test\n */ (-1.5); foo_bar.baz:\"hello\\r\\n\\t\\\"world\\\"\" ] // this line\n#that line";
  94         Tokenizer in(stream);
  95
  96         AssertHasMore(in);
  97         Token token(in.Next());
  98         AssertToken(token.type, token.value, in.Current());
  99         AssertToken(Token::BRACKET_OPEN, token);
 100
 101         AssertHasMore(in);
 102         AssertToken(Token::ANGLE_BRACKET_OPEN, in.Next());
 103         AssertHasMore(in);
 104         AssertToken(Token::NUMBER, "0", in.Next());
 105         AssertHasMore(in);
 106         AssertToken(Token::ANGLE_BRACKET_CLOSE, in.Next());
 107         AssertHasMore(in);
 108         AssertToken(Token::COMMA, in.Next());
 109         AssertHasMore(in);
 110         AssertToken(Token::CHEVRON_OPEN, in.Next());
 111         AssertHasMore(in);
 112         AssertToken(Token::NUMBER, ".5", in.Next());
 113         AssertHasMore(in);
 114         AssertToken(Token::CHEVRON_CLOSE, in.Next());
 115         AssertHasMore(in);
 116         AssertToken(Token::NUMBER, "+3", in.Next());
 117         AssertHasMore(in);
 118         AssertToken(Token::EQUALS, in.Next());
 119         AssertHasMore(in);
 120         AssertToken(Token::COMMENT, "*\n * test\n ", in.Next());
 121         AssertHasMore(in);
 122         AssertToken(Token::PARENTHESIS_OPEN, in.Next());
 123         AssertHasMore(in);
 124         AssertToken(Token::NUMBER, "-1.5", in.Next());
 125         AssertHasMore(in);
 126         AssertToken(Token::PARENTHESIS_CLOSE, in.Next());
 127         AssertHasMore(in);
 128         AssertToken(Token::SEMICOLON, in.Next());
 129         AssertHasMore(in);
 130         AssertToken(Token::IDENTIFIER, "foo_bar.baz", in.Next());
 131         AssertHasMore(in);
 132         AssertToken(Token::COLON, in.Next());
 133         AssertHasMore(in);
 134         AssertToken(Token::STRING, "hello\r\n\t\"world\"", in.Next());
 135         AssertHasMore(in);
 136         AssertToken(Token::BRACKET_CLOSE, in.Next());
 137         AssertHasMore(in);
 138         AssertToken(Token::COMMENT, " this line", in.Next());
 139         AssertHasMore(in);
 140         AssertToken(Token::COMMENT, "that line", in.Next());
 141         CPPUNIT_ASSERT_MESSAGE("expected end of stream", !in.HasMore());
 142         CPPUNIT_ASSERT_THROW_MESSAGE(
 143                 "extracting token after EOS",
 144                 in.Next(), std::runtime_error);
 145 }
 146
 147 void TokenTest::testTokenizerBrokenComment() {
 148         {
 149                 stringstream stream;
 150                 stream << "/* just one more thing…*";
 151                 Tokenizer in(stream);
 152                 AssertHasMore(in);
 153                 CPPUNIT_ASSERT_THROW_MESSAGE(
 154                         "half-closed comment should throw",
 155                         in.Next(), std::runtime_error);
 156         }
 157         {
 158                 stringstream stream;
 159                 stream << "  /";
 160                 Tokenizer in(stream);
 161                 AssertHasMore(in);
 162                 CPPUNIT_ASSERT_THROW_MESSAGE(
 163                         "sole '/' at end of stream should throw",
 164                         in.Next(), std::runtime_error);
 165         }
 166         {
 167                 stringstream stream;
 168                 stream << "/.";
 169                 Tokenizer in(stream);
 170                 AssertHasMore(in);
 171                 CPPUNIT_ASSERT_THROW_MESSAGE(
 172                         "'/' followed by garbage should throw",
 173                         in.Next(), std::runtime_error);
 174         }
 175 }
 176
 177
 178 namespace {
 179
 180 template<class T>
 181 void assert_read(std::string message, T expected, T actual, TokenStreamReader &in) {
 182         stringstream msg;
 183         msg << message << ", current token: " << in.Peek();
 184         CPPUNIT_ASSERT_EQUAL_MESSAGE(
 185                 msg.str(),
 186                 expected, actual);
 187 }
 188
 189 }
 190
 191 void TokenTest::testReader() {
 192         stringstream ss;
 193         ss <<
 194                 "/* booleans */\n"
 195                 "true false yes no on off\n"
 196                 "\"true\" \"false\" \"yes\" \"no\" \"on\" \"off\"\n"
 197                 "1 0 -1\n"
 198                 "# identifiers\n"
 199                 "foo foo_bar vec.y\n"
 200                 "// numbers\n"
 201                 "0 1 +2 -3 4.5\n"
 202                 ".5 1.5 0.25 -1.75 0.625\n"
 203                 "0 1 -1 2.5\n"
 204                 // strings
 205                 "\"hello\" \"\" \"\\r\\n\\t\\\"\"\n"
 206                 "\"world\" foo 12\n"
 207                 // vectors
 208                 "[1,0] [ 0.707, 0.707 ] // vec2\n"
 209                 "[.577,.577 ,0.577] [ 1,-2,3] // vec3\n"
 210                 "[ 0, 0, 0, 1 ] [1,0,0,-1.0] // vec4\n"
 211                 "[640, 480] [3, 4, 5] [0, -10, 100, -1000] # ivecs\n"
 212                 "[ -0.945, 0, -0.326, 0] # quat\n"
 213                 ;
 214         TokenStreamReader in(ss);
 215
 216         // booleans
 217
 218         bool value_bool;
 219         in.ReadBoolean(value_bool);
 220         assert_read("reading boolean true", true, value_bool, in);
 221         in.ReadBoolean(value_bool);
 222         assert_read("reading boolean false", false, value_bool, in);
 223         in.ReadBoolean(value_bool);
 224         assert_read("reading boolean yes", true, value_bool, in);
 225         in.ReadBoolean(value_bool);
 226         assert_read("reading boolean no", false, value_bool, in);
 227         in.ReadBoolean(value_bool);
 228         assert_read("reading boolean on", true, value_bool, in);
 229         in.ReadBoolean(value_bool);
 230         assert_read("reading boolean off", false, value_bool, in);
 231
 232         in.ReadBoolean(value_bool);
 233         assert_read("reading boolean \"true\"", true, value_bool, in);
 234         in.ReadBoolean(value_bool);
 235         assert_read("reading boolean \"false\"", false, value_bool, in);
 236         in.ReadBoolean(value_bool);
 237         assert_read("reading boolean \"yes\"", true, value_bool, in);
 238         in.ReadBoolean(value_bool);
 239         assert_read("reading boolean \"no\"", false, value_bool, in);
 240         in.ReadBoolean(value_bool);
 241         assert_read("reading boolean \"on\"", true, value_bool, in);
 242         in.ReadBoolean(value_bool);
 243         assert_read("reading boolean \"off\"", false, value_bool, in);
 244
 245         in.ReadBoolean(value_bool);
 246         assert_read("reading boolean 1", true, value_bool, in);
 247         in.ReadBoolean(value_bool);
 248         assert_read("reading boolean 0", false, value_bool, in);
 249         in.ReadBoolean(value_bool);
 250         assert_read("reading boolean -1", true, value_bool, in);
 251
 252         // identifiers
 253
 254         string value_ident;
 255         in.ReadIdentifier(value_ident);
 256         assert_read<string>("reading identifier foo", "foo", value_ident, in);
 257         in.ReadIdentifier(value_ident);
 258         assert_read<string>("reading identifier foo_bar", "foo_bar", value_ident, in);
 259         in.ReadIdentifier(value_ident);
 260         assert_read<string>("reading identifier vec.y", "vec.y", value_ident, in);
 261
 262         // numbers
 263         int value_int;
 264         in.ReadNumber(value_int);
 265         assert_read("reading integer 0", 0, value_int, in);
 266         in.ReadNumber(value_int);
 267         assert_read("reading integer 1", 1, value_int, in);
 268         in.ReadNumber(value_int);
 269         assert_read("reading integer +2", 2, value_int, in);
 270         in.ReadNumber(value_int);
 271         assert_read("reading integer -3", -3, value_int, in);
 272         in.ReadNumber(value_int);
 273         assert_read("reading integer 4.5", 4, value_int, in);
 274
 275         float value_float;
 276         in.ReadNumber(value_float);
 277         assert_read("reading float .5", .5f, value_float, in);
 278         in.ReadNumber(value_float);
 279         assert_read("reading float 1.5", 1.5f, value_float, in);
 280         in.ReadNumber(value_float);
 281         assert_read("reading float 0.25", .25f, value_float, in);
 282         in.ReadNumber(value_float);
 283         assert_read("reading float -1.75", -1.75f, value_float, in);
 284         in.ReadNumber(value_float);
 285         assert_read("reading float 0.625", 0.625f, value_float, in);
 286
 287         unsigned long value_uint;
 288         in.ReadNumber(value_uint);
 289         assert_read("reading unsigned integer 0", 0ul, value_uint, in);
 290         in.ReadNumber(value_uint);
 291         assert_read("reading unsigned integer 1", 1ul, value_uint, in);
 292         in.ReadNumber(value_uint);
 293         assert_read("reading unsigned integer -1", -1ul, value_uint, in);
 294         in.ReadNumber(value_uint);
 295         assert_read("reading unsigned integer 2.5", 2ul, value_uint, in);
 296
 297         // strings
 298
 299         string value_string;
 300         in.ReadString(value_string);
 301         assert_read<string>(
 302                 "reading string \"hello\"",
 303                 "hello", value_string, in);
 304         in.ReadString(value_string);
 305         assert_read<string>(
 306                 "reading string \"\"",
 307                 "", value_string, in);
 308         in.ReadString(value_string);
 309         assert_read<string>(
 310                 "reading string \"\\r\\n\\t\\\"\"",
 311                 "\r\n\t\"", value_string, in);
 312
 313         in.ReadRelaxedString(value_string);
 314         assert_read<string>(
 315                 "reading relaxed string \"world\"",
 316                 "world", value_string, in);
 317
 318         in.ReadRelaxedString(value_string);
 319         assert_read<string>(
 320                 "reading relaxed string foo",
 321                 "foo", value_string, in);
 322
 323         in.ReadRelaxedString(value_string);
 324         assert_read<string>(
 325                 "reading relaxed string 12",
 326                 "12", value_string, in);
 327
 328         // vectors
 329
 330         glm::vec2 value_vec2;
 331         in.ReadVec(value_vec2);
 332         assert_read(
 333                 "reading vector [1,0]",
 334                 glm::vec2(1, 0), value_vec2, in);
 335         in.ReadVec(value_vec2);
 336         assert_read(
 337                 "reading vector [ 0.707, 0.707 ]",
 338                 glm::vec2(.707, .707), value_vec2, in);
 339
 340         glm::vec3 value_vec3;
 341         in.ReadVec(value_vec3);
 342         assert_read(
 343                 "reading vector [.577,.577 ,0.577]",
 344                 glm::vec3(.577, .577, .577), value_vec3, in);
 345         in.ReadVec(value_vec3);
 346         assert_read(
 347                 "reading vector [ 1,-2,3]",
 348                 glm::vec3(1, -2, 3), value_vec3, in);
 349
 350         glm::vec4 value_vec4;
 351         in.ReadVec(value_vec4);
 352         assert_read(
 353                 "reading vector [ 0, 0, 0, 1 ]",
 354                 glm::vec4(0, 0, 0, 1), value_vec4, in);
 355         in.ReadVec(value_vec4);
 356         assert_read(
 357                 "reading vector [1,0,0,-1.0]",
 358                 glm::vec4(1, 0, 0, -1), value_vec4, in);
 359
 360         glm::ivec2 value_ivec2;
 361         in.ReadVec(value_ivec2);
 362         assert_read(
 363                 "reading integer vector [640, 480]",
 364                 glm::ivec2(640, 480), value_ivec2, in);
 365         glm::ivec3 value_ivec3;
 366         in.ReadVec(value_ivec3);
 367         assert_read(
 368                 "reading integer vector [3, 4, 5]",
 369                 glm::ivec3(3, 4, 5), value_ivec3, in);
 370         glm::ivec4 value_ivec4;
 371         in.ReadVec(value_ivec4);
 372         assert_read(
 373                 "reading integer vector [0, -10, 100, -1000]",
 374                 glm::ivec4(0, -10, 100, -1000), value_ivec4, in);
 375
 376         glm::quat value_quat;
 377         in.ReadQuat(value_quat);
 378         assert_read(
 379                 "reading quaternion [ -0.945, 0, -0.326, 0]",
 380                 glm::quat(-0.945, 0, -0.326, 0), value_quat, in);
 381         // TODO: comment at end of stream makes it think there's more?
 382         //CPPUNIT_ASSERT_MESSAGE("expected end of stream", !in.HasMore());
 383         // TODO: and it even works??
 384         //CPPUNIT_ASSERT_THROW_MESSAGE(
 385         //      "extracting token after EOS",
 386         //      in.Next(), std::runtime_error);
 387 }
 388
 389 void TokenTest::testReaderEmpty() {
 390         { // zero length stream
 391                 stringstream ss;
 392                 ss << "";
 393                 TokenStreamReader in(ss);
 394                 CPPUNIT_ASSERT_MESSAGE(
 395                         "empty stream shouldn't have tokens",
 396                         !in.HasMore());
 397         }
 398         { // stream consisting solely of comments
 399                 stringstream ss;
 400                 ss <<
 401                         "/*\n"
 402                         " * hello\n"
 403                         " */\n"
 404                         "#hello\n"
 405                         "// is there anybody out there\n"
 406                         ;
 407                 TokenStreamReader in(ss);
 408                 CPPUNIT_ASSERT_MESSAGE(
 409                         "comment stream shouldn't have tokens",
 410                         !in.HasMore());
 411         }
 412 }
 413
 414 void TokenTest::testReaderMalformed() {
 415         {
 416                 stringstream ss;
 417                 ss << "a";
 418                 TokenStreamReader in(ss);
 419                 CPPUNIT_ASSERT_THROW_MESSAGE(
 420                         "unexpected token type should throw",
 421                         in.GetInt(), std::runtime_error);
 422         }
 423         {
 424                 stringstream ss;
 425                 ss << ":";
 426                 TokenStreamReader in(ss);
 427                 CPPUNIT_ASSERT_THROW_MESSAGE(
 428                         "casting ':' to bool should throw",
 429                         in.GetBool(), std::runtime_error);
 430         }
 431         {
 432                 stringstream ss;
 433                 ss << "hello";
 434                 TokenStreamReader in(ss);
 435                 CPPUNIT_ASSERT_THROW_MESSAGE(
 436                         "casting \"hello\" to bool should throw",
 437                         in.GetBool(), std::runtime_error);
 438         }
 439 }
 440
 441
 442 void TokenTest::AssertStreamOutput(
 443         Token::Type t,
 444         string expected
 445 ) {
 446         stringstream conv;
 447         conv << t;
 448         CPPUNIT_ASSERT_EQUAL_MESSAGE(
 449                 "unexpected std::ostream << Token::Type result",
 450                 expected, conv.str());
 451 }
 452
 453 void TokenTest::AssertStreamOutput(
 454         const Token &t,
 455         string expected
 456 ) {
 457         stringstream conv;
 458         conv << t;
 459         CPPUNIT_ASSERT_EQUAL_MESSAGE(
 460                 "unexpected std::ostream << Token result",
 461                 expected, conv.str());
 462 }
 463
 464 void TokenTest::AssertHasMore(Tokenizer &in) {
 465         CPPUNIT_ASSERT_MESSAGE("unexpected end of stream", in.HasMore());
 466 }
 467
 468 void TokenTest::AssertToken(
 469         Token::Type expected_type,
 470         const Token &actual_token
 471 ) {
 472         AssertToken(expected_type, "", actual_token);
 473 }
 474
 475 void TokenTest::AssertToken(
 476         Token::Type expected_type,
 477         string expected_value,
 478         const Token &actual_token
 479 ) {
 480         CPPUNIT_ASSERT_EQUAL_MESSAGE(
 481                 "unexpected token type",
 482                 expected_type, actual_token.type);
 483         CPPUNIT_ASSERT_EQUAL_MESSAGE(
 484                 "unexpected token value",
 485                 expected_value, actual_token.value);
 486 }
 487
 488 }
 489 }
 490 }