tst/io/TokenTest.cpp

   1 #include "TokenTest.hpp"
   2
   3 #include "io/TokenStreamReader.hpp"
   4
   5 #include <sstream>
   6 #include <stdexcept>
   7 #include <glm/gtx/io.hpp>
   8
   9 CPPUNIT_TEST_SUITE_REGISTRATION(blank::test::TokenTest);
  10
  11 using namespace std;
  12
  13 namespace blank {
  14 namespace test {
  15
  16 void TokenTest::setUp() {
  17
  18 }
  19
  20 void TokenTest::tearDown() {
  21
  22 }
  23
  24
  25 void TokenTest::testTypeIO() {
  26         AssertStreamOutput(Token::UNKNOWN, "UNKNOWN");
  27         AssertStreamOutput(Token::ANGLE_BRACKET_OPEN, "ANGLE_BRACKET_OPEN");
  28         AssertStreamOutput(Token::ANGLE_BRACKET_CLOSE, "ANGLE_BRACKET_CLOSE");
  29         AssertStreamOutput(Token::CHEVRON_OPEN, "CHEVRON_OPEN");
  30         AssertStreamOutput(Token::CHEVRON_CLOSE, "CHEVRON_CLOSE");
  31         AssertStreamOutput(Token::BRACKET_OPEN, "BRACKET_OPEN");
  32         AssertStreamOutput(Token::BRACKET_CLOSE, "BRACKET_CLOSE");
  33         AssertStreamOutput(Token::PARENTHESIS_OPEN, "PARENTHESIS_OPEN");
  34         AssertStreamOutput(Token::PARENTHESIS_CLOSE, "PARENTHESIS_CLOSE");
  35         AssertStreamOutput(Token::COLON, "COLON");
  36         AssertStreamOutput(Token::SEMICOLON, "SEMICOLON");
  37         AssertStreamOutput(Token::COMMA, "COMMA");
  38         AssertStreamOutput(Token::EQUALS, "EQUALS");
  39         AssertStreamOutput(Token::NUMBER, "NUMBER");
  40         AssertStreamOutput(Token::STRING, "STRING");
  41         AssertStreamOutput(Token::IDENTIFIER, "IDENTIFIER");
  42         AssertStreamOutput(Token::COMMENT, "COMMENT");
  43 }
  44
  45 void TokenTest::testTokenIO() {
  46         Token t;
  47         t.value = "why oh why";
  48         AssertStreamOutput(t, "UNKNOWN(why oh why)");
  49         t.type = Token::UNKNOWN;
  50         t.value = "do I have no purpose";
  51         AssertStreamOutput(t, "UNKNOWN(do I have no purpose)");
  52         t.type = Token::ANGLE_BRACKET_OPEN;
  53         AssertStreamOutput(t, "ANGLE_BRACKET_OPEN");
  54         t.type = Token::ANGLE_BRACKET_CLOSE;
  55         AssertStreamOutput(t, "ANGLE_BRACKET_CLOSE");
  56         t.type = Token::CHEVRON_OPEN;
  57         AssertStreamOutput(t, "CHEVRON_OPEN");
  58         t.type = Token::CHEVRON_CLOSE;
  59         AssertStreamOutput(t, "CHEVRON_CLOSE");
  60         t.type = Token::BRACKET_OPEN;
  61         AssertStreamOutput(t, "BRACKET_OPEN");
  62         t.type = Token::BRACKET_CLOSE;
  63         AssertStreamOutput(t, "BRACKET_CLOSE");
  64         t.type = Token::PARENTHESIS_OPEN;
  65         AssertStreamOutput(t, "PARENTHESIS_OPEN");
  66         t.type = Token::PARENTHESIS_CLOSE;
  67         AssertStreamOutput(t, "PARENTHESIS_CLOSE");
  68         t.type = Token::COLON;
  69         AssertStreamOutput(t, "COLON");
  70         t.type = Token::SEMICOLON;
  71         AssertStreamOutput(t, "SEMICOLON");
  72         t.type = Token::COMMA;
  73         AssertStreamOutput(t, "COMMA");
  74         t.type = Token::EQUALS;
  75         AssertStreamOutput(t, "EQUALS");
  76         t.type = Token::NUMBER;
  77         t.value = "15";
  78         AssertStreamOutput(t, "NUMBER(15)");
  79         t.type = Token::STRING;
  80         t.value = "hello world";
  81         AssertStreamOutput(t, "STRING(hello world)");
  82         t.type = Token::IDENTIFIER;
  83         t.value = "foo";
  84         AssertStreamOutput(t, "IDENTIFIER(foo)");
  85         t.type = Token::COMMENT;
  86         t.value = "WITHOUT ANY WARRANTY";
  87         AssertStreamOutput(t, "COMMENT(WITHOUT ANY WARRANTY)");
  88 }
  89
  90 void TokenTest::testTokenizer() {
  91         stringstream stream;
  92         stream << "[{0},<.5>+3=/**\n * test\n */ (-1.5); foo_bar.baz:\"hello\\r\\n\\t\\\"world\\\"\" ] // this line\n#that line";
  93         Tokenizer in(stream);
  94
  95         AssertHasMore(in);
  96         Token token(in.Next());
  97         AssertToken(token.type, token.value, in.Current());
  98         AssertToken(Token::BRACKET_OPEN, token);
  99
 100         AssertHasMore(in);
 101         AssertToken(Token::ANGLE_BRACKET_OPEN, in.Next());
 102         AssertHasMore(in);
 103         AssertToken(Token::NUMBER, "0", in.Next());
 104         AssertHasMore(in);
 105         AssertToken(Token::ANGLE_BRACKET_CLOSE, in.Next());
 106         AssertHasMore(in);
 107         AssertToken(Token::COMMA, in.Next());
 108         AssertHasMore(in);
 109         AssertToken(Token::CHEVRON_OPEN, in.Next());
 110         AssertHasMore(in);
 111         AssertToken(Token::NUMBER, ".5", in.Next());
 112         AssertHasMore(in);
 113         AssertToken(Token::CHEVRON_CLOSE, in.Next());
 114         AssertHasMore(in);
 115         AssertToken(Token::NUMBER, "+3", in.Next());
 116         AssertHasMore(in);
 117         AssertToken(Token::EQUALS, in.Next());
 118         AssertHasMore(in);
 119         AssertToken(Token::COMMENT, "*\n * test\n ", in.Next());
 120         AssertHasMore(in);
 121         AssertToken(Token::PARENTHESIS_OPEN, in.Next());
 122         AssertHasMore(in);
 123         AssertToken(Token::NUMBER, "-1.5", in.Next());
 124         AssertHasMore(in);
 125         AssertToken(Token::PARENTHESIS_CLOSE, in.Next());
 126         AssertHasMore(in);
 127         AssertToken(Token::SEMICOLON, in.Next());
 128         AssertHasMore(in);
 129         AssertToken(Token::IDENTIFIER, "foo_bar.baz", in.Next());
 130         AssertHasMore(in);
 131         AssertToken(Token::COLON, in.Next());
 132         AssertHasMore(in);
 133         AssertToken(Token::STRING, "hello\r\n\t\"world\"", in.Next());
 134         AssertHasMore(in);
 135         AssertToken(Token::BRACKET_CLOSE, in.Next());
 136         AssertHasMore(in);
 137         AssertToken(Token::COMMENT, " this line", in.Next());
 138         AssertHasMore(in);
 139         AssertToken(Token::COMMENT, "that line", in.Next());
 140         CPPUNIT_ASSERT_MESSAGE("expected end of stream", !in.HasMore());
 141         CPPUNIT_ASSERT_THROW_MESSAGE(
 142                 "extracting token after EOS",
 143                 in.Next(), std::runtime_error);
 144 }
 145
 146 void TokenTest::testTokenizerBrokenComment() {
 147         {
 148                 stringstream stream;
 149                 stream << "/* just one more thing…*";
 150                 Tokenizer in(stream);
 151                 AssertHasMore(in);
 152                 CPPUNIT_ASSERT_THROW_MESSAGE(
 153                         "half-closed comment should throw",
 154                         in.Next(), std::runtime_error);
 155         }
 156         {
 157                 stringstream stream;
 158                 stream << "  /";
 159                 Tokenizer in(stream);
 160                 AssertHasMore(in);
 161                 CPPUNIT_ASSERT_THROW_MESSAGE(
 162                         "sole '/' at end of stream should throw",
 163                         in.Next(), std::runtime_error);
 164         }
 165         {
 166                 stringstream stream;
 167                 stream << "/.";
 168                 Tokenizer in(stream);
 169                 AssertHasMore(in);
 170                 CPPUNIT_ASSERT_THROW_MESSAGE(
 171                         "'/' followed by garbage should throw",
 172                         in.Next(), std::runtime_error);
 173         }
 174 }
 175
 176
 177 namespace {
 178
 179 template<class T>
 180 void assert_read(std::string message, T expected, T actual, TokenStreamReader &in) {
 181         stringstream msg;
 182         msg << message << ", current token: " << in.Peek();
 183         CPPUNIT_ASSERT_EQUAL_MESSAGE(
 184                 msg.str(),
 185                 expected, actual);
 186 }
 187
 188 }
 189
 190 void TokenTest::testReader() {
 191         stringstream ss;
 192         ss <<
 193                 "/* booleans */\n"
 194                 "true false yes no on off\n"
 195                 "\"true\" \"false\" \"yes\" \"no\" \"on\" \"off\"\n"
 196                 "1 0 -1\n"
 197                 "# identifiers\n"
 198                 "foo foo_bar vec.y\n"
 199                 "// numbers\n"
 200                 "0 1 +2 -3 4.5\n"
 201                 ".5 1.5 0.25 -1.75 0.625\n"
 202                 "0 1 -1 2.5\n"
 203                 // strings
 204                 "\"hello\" \"\" \"\\r\\n\\t\\\"\"\n"
 205                 // vectors
 206                 "[1,0] [ 0.707, 0.707 ] // vec2\n"
 207                 "[.577,.577 ,0.577] [ 1,-2,3] // vec3\n"
 208                 "[ 0, 0, 0, 1 ] [1,0,0,-1.0] // vec4\n"
 209                 "[640, 480] [3, 4, 5] [0, -10, 100, -1000] # ivecs\n"
 210                 "[ -0.945, 0, -0.326, 0] # quat\n"
 211                 ;
 212         TokenStreamReader in(ss);
 213
 214         // booleans
 215
 216         bool value_bool;
 217         in.ReadBoolean(value_bool);
 218         assert_read("reading boolean true", true, value_bool, in);
 219         in.ReadBoolean(value_bool);
 220         assert_read("reading boolean false", false, value_bool, in);
 221         in.ReadBoolean(value_bool);
 222         assert_read("reading boolean yes", true, value_bool, in);
 223         in.ReadBoolean(value_bool);
 224         assert_read("reading boolean no", false, value_bool, in);
 225         in.ReadBoolean(value_bool);
 226         assert_read("reading boolean on", true, value_bool, in);
 227         in.ReadBoolean(value_bool);
 228         assert_read("reading boolean off", false, value_bool, in);
 229
 230         in.ReadBoolean(value_bool);
 231         assert_read("reading boolean \"true\"", true, value_bool, in);
 232         in.ReadBoolean(value_bool);
 233         assert_read("reading boolean \"false\"", false, value_bool, in);
 234         in.ReadBoolean(value_bool);
 235         assert_read("reading boolean \"yes\"", true, value_bool, in);
 236         in.ReadBoolean(value_bool);
 237         assert_read("reading boolean \"no\"", false, value_bool, in);
 238         in.ReadBoolean(value_bool);
 239         assert_read("reading boolean \"on\"", true, value_bool, in);
 240         in.ReadBoolean(value_bool);
 241         assert_read("reading boolean \"off\"", false, value_bool, in);
 242
 243         in.ReadBoolean(value_bool);
 244         assert_read("reading boolean 1", true, value_bool, in);
 245         in.ReadBoolean(value_bool);
 246         assert_read("reading boolean 0", false, value_bool, in);
 247         in.ReadBoolean(value_bool);
 248         assert_read("reading boolean -1", true, value_bool, in);
 249
 250         // identifiers
 251
 252         string value_ident;
 253         in.ReadIdentifier(value_ident);
 254         assert_read<string>("reading identifier foo", "foo", value_ident, in);
 255         in.ReadIdentifier(value_ident);
 256         assert_read<string>("reading identifier foo_bar", "foo_bar", value_ident, in);
 257         in.ReadIdentifier(value_ident);
 258         assert_read<string>("reading identifier vec.y", "vec.y", value_ident, in);
 259
 260         // numbers
 261         int value_int;
 262         in.ReadNumber(value_int);
 263         assert_read("reading integer 0", 0, value_int, in);
 264         in.ReadNumber(value_int);
 265         assert_read("reading integer 1", 1, value_int, in);
 266         in.ReadNumber(value_int);
 267         assert_read("reading integer +2", 2, value_int, in);
 268         in.ReadNumber(value_int);
 269         assert_read("reading integer -3", -3, value_int, in);
 270         in.ReadNumber(value_int);
 271         assert_read("reading integer 4.5", 4, value_int, in);
 272
 273         float value_float;
 274         in.ReadNumber(value_float);
 275         assert_read("reading float .5", .5f, value_float, in);
 276         in.ReadNumber(value_float);
 277         assert_read("reading float 1.5", 1.5f, value_float, in);
 278         in.ReadNumber(value_float);
 279         assert_read("reading float 0.25", .25f, value_float, in);
 280         in.ReadNumber(value_float);
 281         assert_read("reading float -1.75", -1.75f, value_float, in);
 282         in.ReadNumber(value_float);
 283         assert_read("reading float 0.625", 0.625f, value_float, in);
 284
 285         unsigned long value_uint;
 286         in.ReadNumber(value_uint);
 287         assert_read("reading unsigned integer 0", 0ul, value_uint, in);
 288         in.ReadNumber(value_uint);
 289         assert_read("reading unsigned integer 1", 1ul, value_uint, in);
 290         in.ReadNumber(value_uint);
 291         assert_read("reading unsigned integer -1", -1ul, value_uint, in);
 292         in.ReadNumber(value_uint);
 293         assert_read("reading unsigned integer 2.5", 2ul, value_uint, in);
 294
 295         // strings
 296
 297         string value_string;
 298         in.ReadString(value_string);
 299         assert_read<string>(
 300                 "reading string \"hello\"",
 301                 "hello", value_string, in);
 302         in.ReadString(value_string);
 303         assert_read<string>(
 304                 "reading string \"\"",
 305                 "", value_string, in);
 306         in.ReadString(value_string);
 307         assert_read<string>(
 308                 "reading string \"\\r\\n\\t\\\"\"",
 309                 "\r\n\t\"", value_string, in);
 310
 311         // vectors
 312
 313         glm::vec2 value_vec2;
 314         in.ReadVec(value_vec2);
 315         assert_read(
 316                 "reading vector [1,0]",
 317                 glm::vec2(1, 0), value_vec2, in);
 318         in.ReadVec(value_vec2);
 319         assert_read(
 320                 "reading vector [ 0.707, 0.707 ]",
 321                 glm::vec2(.707, .707), value_vec2, in);
 322
 323         glm::vec3 value_vec3;
 324         in.ReadVec(value_vec3);
 325         assert_read(
 326                 "reading vector [.577,.577 ,0.577]",
 327                 glm::vec3(.577, .577, .577), value_vec3, in);
 328         in.ReadVec(value_vec3);
 329         assert_read(
 330                 "reading vector [ 1,-2,3]",
 331                 glm::vec3(1, -2, 3), value_vec3, in);
 332
 333         glm::vec4 value_vec4;
 334         in.ReadVec(value_vec4);
 335         assert_read(
 336                 "reading vector [ 0, 0, 0, 1 ]",
 337                 glm::vec4(0, 0, 0, 1), value_vec4, in);
 338         in.ReadVec(value_vec4);
 339         assert_read(
 340                 "reading vector [1,0,0,-1.0]",
 341                 glm::vec4(1, 0, 0, -1), value_vec4, in);
 342
 343         glm::ivec2 value_ivec2;
 344         in.ReadVec(value_ivec2);
 345         assert_read(
 346                 "reading integer vector [640, 480]",
 347                 glm::ivec2(640, 480), value_ivec2, in);
 348         glm::ivec3 value_ivec3;
 349         in.ReadVec(value_ivec3);
 350         assert_read(
 351                 "reading integer vector [3, 4, 5]",
 352                 glm::ivec3(3, 4, 5), value_ivec3, in);
 353         glm::ivec4 value_ivec4;
 354         in.ReadVec(value_ivec4);
 355         assert_read(
 356                 "reading integer vector [0, -10, 100, -1000]",
 357                 glm::ivec4(0, -10, 100, -1000), value_ivec4, in);
 358
 359         glm::quat value_quat;
 360         in.ReadQuat(value_quat);
 361         assert_read(
 362                 "reading quaternion [ -0.945, 0, -0.326, 0]",
 363                 glm::quat(-0.945, 0, -0.326, 0), value_quat, in);
 364         // TODO: comment at end of stream makes it think there's more?
 365         //CPPUNIT_ASSERT_MESSAGE("expected end of stream", !in.HasMore());
 366         // TODO: and it even works??
 367         //CPPUNIT_ASSERT_THROW_MESSAGE(
 368         //      "extracting token after EOS",
 369         //      in.Next(), std::runtime_error);
 370 }
 371
 372 void TokenTest::testReaderEmpty() {
 373         { // zero length stream
 374                 stringstream ss;
 375                 ss << "";
 376                 TokenStreamReader in(ss);
 377                 CPPUNIT_ASSERT_MESSAGE(
 378                         "empty stream shouldn't have tokens",
 379                         !in.HasMore());
 380         }
 381         { // stream consisting solely of comments
 382                 stringstream ss;
 383                 ss <<
 384                         "/*\n"
 385                         " * hello\n"
 386                         " */\n"
 387                         "#hello\n"
 388                         "// is there anybody out there\n"
 389                         ;
 390                 TokenStreamReader in(ss);
 391                 CPPUNIT_ASSERT_MESSAGE(
 392                         "comment stream shouldn't have tokens",
 393                         !in.HasMore());
 394         }
 395 }
 396
 397 void TokenTest::testReaderMalformed() {
 398         {
 399                 stringstream ss;
 400                 ss << "a";
 401                 TokenStreamReader in(ss);
 402                 CPPUNIT_ASSERT_THROW_MESSAGE(
 403                         "unexpected token type should throw",
 404                         in.GetInt(), std::runtime_error);
 405         }
 406         {
 407                 stringstream ss;
 408                 ss << ":";
 409                 TokenStreamReader in(ss);
 410                 CPPUNIT_ASSERT_THROW_MESSAGE(
 411                         "casting ':' to bool should throw",
 412                         in.GetBool(), std::runtime_error);
 413         }
 414         {
 415                 stringstream ss;
 416                 ss << "hello";
 417                 TokenStreamReader in(ss);
 418                 CPPUNIT_ASSERT_THROW_MESSAGE(
 419                         "casting \"hello\" to bool should throw",
 420                         in.GetBool(), std::runtime_error);
 421         }
 422 }
 423
 424
 425 void TokenTest::AssertStreamOutput(
 426         Token::Type t,
 427         string expected
 428 ) {
 429         stringstream conv;
 430         conv << t;
 431         CPPUNIT_ASSERT_EQUAL_MESSAGE(
 432                 "unexpected std::ostream << Token::Type result",
 433                 expected, conv.str());
 434 }
 435
 436 void TokenTest::AssertStreamOutput(
 437         const Token &t,
 438         string expected
 439 ) {
 440         stringstream conv;
 441         conv << t;
 442         CPPUNIT_ASSERT_EQUAL_MESSAGE(
 443                 "unexpected std::ostream << Token result",
 444                 expected, conv.str());
 445 }
 446
 447 void TokenTest::AssertHasMore(Tokenizer &in) {
 448         CPPUNIT_ASSERT_MESSAGE("unexpected end of stream", in.HasMore());
 449 }
 450
 451 void TokenTest::AssertToken(
 452         Token::Type expected_type,
 453         const Token &actual_token
 454 ) {
 455         AssertToken(expected_type, "", actual_token);
 456 }
 457
 458 void TokenTest::AssertToken(
 459         Token::Type expected_type,
 460         string expected_value,
 461         const Token &actual_token
 462 ) {
 463         CPPUNIT_ASSERT_EQUAL_MESSAGE(
 464                 "unexpected token type",
 465                 expected_type, actual_token.type);
 466         CPPUNIT_ASSERT_EQUAL_MESSAGE(
 467                 "unexpected token value",
 468                 expected_value, actual_token.value);
 469 }
 470
 471 }
 472 }