tst/io/TokenTest.cpp

   1 #include "TokenTest.hpp"
   2
   3 #include "io/TokenStreamReader.hpp"
   4
   5 #include <sstream>
   6 #include <stdexcept>
   7 #include <glm/gtx/io.hpp>
   8
   9 CPPUNIT_TEST_SUITE_REGISTRATION(blobs::io::test::TokenTest);
  10
  11 using namespace std;
  12
  13 namespace blobs {
  14 namespace io {
  15 namespace test {
  16
  17 void TokenTest::setUp() {
  18
  19 }
  20
  21 void TokenTest::tearDown() {
  22
  23 }
  24
  25
  26 void TokenTest::testTypeIO() {
  27         AssertStreamOutput(Token::UNKNOWN, "UNKNOWN");
  28         AssertStreamOutput(Token::ANGLE_BRACKET_OPEN, "ANGLE_BRACKET_OPEN");
  29         AssertStreamOutput(Token::ANGLE_BRACKET_CLOSE, "ANGLE_BRACKET_CLOSE");
  30         AssertStreamOutput(Token::CHEVRON_OPEN, "CHEVRON_OPEN");
  31         AssertStreamOutput(Token::CHEVRON_CLOSE, "CHEVRON_CLOSE");
  32         AssertStreamOutput(Token::BRACKET_OPEN, "BRACKET_OPEN");
  33         AssertStreamOutput(Token::BRACKET_CLOSE, "BRACKET_CLOSE");
  34         AssertStreamOutput(Token::PARENTHESIS_OPEN, "PARENTHESIS_OPEN");
  35         AssertStreamOutput(Token::PARENTHESIS_CLOSE, "PARENTHESIS_CLOSE");
  36         AssertStreamOutput(Token::COLON, "COLON");
  37         AssertStreamOutput(Token::SEMICOLON, "SEMICOLON");
  38         AssertStreamOutput(Token::COMMA, "COMMA");
  39         AssertStreamOutput(Token::EQUALS, "EQUALS");
  40         AssertStreamOutput(Token::NUMBER, "NUMBER");
  41         AssertStreamOutput(Token::STRING, "STRING");
  42         AssertStreamOutput(Token::IDENTIFIER, "IDENTIFIER");
  43         AssertStreamOutput(Token::COMMENT, "COMMENT");
  44 }
  45
  46 void TokenTest::testTokenIO() {
  47         Token t;
  48         t.value = "why oh why";
  49         AssertStreamOutput(t, "UNKNOWN(why oh why)");
  50         t.type = Token::UNKNOWN;
  51         t.value = "do I have no purpose";
  52         AssertStreamOutput(t, "UNKNOWN(do I have no purpose)");
  53         t.type = Token::ANGLE_BRACKET_OPEN;
  54         AssertStreamOutput(t, "ANGLE_BRACKET_OPEN");
  55         t.type = Token::ANGLE_BRACKET_CLOSE;
  56         AssertStreamOutput(t, "ANGLE_BRACKET_CLOSE");
  57         t.type = Token::CHEVRON_OPEN;
  58         AssertStreamOutput(t, "CHEVRON_OPEN");
  59         t.type = Token::CHEVRON_CLOSE;
  60         AssertStreamOutput(t, "CHEVRON_CLOSE");
  61         t.type = Token::BRACKET_OPEN;
  62         AssertStreamOutput(t, "BRACKET_OPEN");
  63         t.type = Token::BRACKET_CLOSE;
  64         AssertStreamOutput(t, "BRACKET_CLOSE");
  65         t.type = Token::PARENTHESIS_OPEN;
  66         AssertStreamOutput(t, "PARENTHESIS_OPEN");
  67         t.type = Token::PARENTHESIS_CLOSE;
  68         AssertStreamOutput(t, "PARENTHESIS_CLOSE");
  69         t.type = Token::COLON;
  70         AssertStreamOutput(t, "COLON");
  71         t.type = Token::SEMICOLON;
  72         AssertStreamOutput(t, "SEMICOLON");
  73         t.type = Token::COMMA;
  74         AssertStreamOutput(t, "COMMA");
  75         t.type = Token::EQUALS;
  76         AssertStreamOutput(t, "EQUALS");
  77         t.type = Token::NUMBER;
  78         t.value = "15";
  79         AssertStreamOutput(t, "NUMBER(15)");
  80         t.type = Token::STRING;
  81         t.value = "hello world";
  82         AssertStreamOutput(t, "STRING(hello world)");
  83         t.type = Token::IDENTIFIER;
  84         t.value = "foo";
  85         AssertStreamOutput(t, "IDENTIFIER(foo)");
  86         t.type = Token::COMMENT;
  87         t.value = "WITHOUT ANY WARRANTY";
  88         AssertStreamOutput(t, "COMMENT(WITHOUT ANY WARRANTY)");
  89 }
  90
  91 void TokenTest::testTokenizer() {
  92         stringstream stream;
  93         stream << "[{0},<.5>+3=/**\n * test\n */ (-1.5); foo_bar.baz:\"hello\\r\\n\\t\\\"world\\\"\" ] // this line\n#that line";
  94         Tokenizer in(stream);
  95
  96         AssertHasMore(in);
  97         Token token(in.Next());
  98         AssertToken(token.type, token.value, in.Current());
  99         AssertToken(Token::BRACKET_OPEN, token);
 100
 101         AssertHasMore(in);
 102         AssertToken(Token::ANGLE_BRACKET_OPEN, in.Next());
 103         AssertHasMore(in);
 104         AssertToken(Token::NUMBER, "0", in.Next());
 105         AssertHasMore(in);
 106         AssertToken(Token::ANGLE_BRACKET_CLOSE, in.Next());
 107         AssertHasMore(in);
 108         AssertToken(Token::COMMA, in.Next());
 109         AssertHasMore(in);
 110         AssertToken(Token::CHEVRON_OPEN, in.Next());
 111         AssertHasMore(in);
 112         AssertToken(Token::NUMBER, ".5", in.Next());
 113         AssertHasMore(in);
 114         AssertToken(Token::CHEVRON_CLOSE, in.Next());
 115         AssertHasMore(in);
 116         AssertToken(Token::NUMBER, "+3", in.Next());
 117         AssertHasMore(in);
 118         AssertToken(Token::EQUALS, in.Next());
 119         AssertHasMore(in);
 120         AssertToken(Token::COMMENT, "*\n * test\n ", in.Next());
 121         AssertHasMore(in);
 122         AssertToken(Token::PARENTHESIS_OPEN, in.Next());
 123         AssertHasMore(in);
 124         AssertToken(Token::NUMBER, "-1.5", in.Next());
 125         AssertHasMore(in);
 126         AssertToken(Token::PARENTHESIS_CLOSE, in.Next());
 127         AssertHasMore(in);
 128         AssertToken(Token::SEMICOLON, in.Next());
 129         AssertHasMore(in);
 130         AssertToken(Token::IDENTIFIER, "foo_bar.baz", in.Next());
 131         AssertHasMore(in);
 132         AssertToken(Token::COLON, in.Next());
 133         AssertHasMore(in);
 134         AssertToken(Token::STRING, "hello\r\n\t\"world\"", in.Next());
 135         AssertHasMore(in);
 136         AssertToken(Token::BRACKET_CLOSE, in.Next());
 137         AssertHasMore(in);
 138         AssertToken(Token::COMMENT, " this line", in.Next());
 139         AssertHasMore(in);
 140         AssertToken(Token::COMMENT, "that line", in.Next());
 141         CPPUNIT_ASSERT_MESSAGE("expected end of stream", !in.HasMore());
 142         CPPUNIT_ASSERT_THROW_MESSAGE(
 143                 "extracting token after EOS",
 144                 in.Next(), std::runtime_error);
 145 }
 146
 147 void TokenTest::testTokenizerBrokenComment() {
 148         {
 149                 stringstream stream;
 150                 stream << "/* just one more thing…*";
 151                 Tokenizer in(stream);
 152                 AssertHasMore(in);
 153                 CPPUNIT_ASSERT_THROW_MESSAGE(
 154                         "half-closed comment should throw",
 155                         in.Next(), std::runtime_error);
 156         }
 157         {
 158                 stringstream stream;
 159                 stream << "  /";
 160                 Tokenizer in(stream);
 161                 AssertHasMore(in);
 162                 CPPUNIT_ASSERT_THROW_MESSAGE(
 163                         "sole '/' at end of stream should throw",
 164                         in.Next(), std::runtime_error);
 165         }
 166         {
 167                 stringstream stream;
 168                 stream << "/.";
 169                 Tokenizer in(stream);
 170                 AssertHasMore(in);
 171                 CPPUNIT_ASSERT_THROW_MESSAGE(
 172                         "'/' followed by garbage should throw",
 173                         in.Next(), std::runtime_error);
 174         }
 175 }
 176
 177
 178 namespace {
 179
 180 template<class T>
 181 void assert_read(std::string message, T expected, T actual, TokenStreamReader &in) {
 182         stringstream msg;
 183         msg << message << ", current token: " << in.Peek();
 184         CPPUNIT_ASSERT_EQUAL_MESSAGE(
 185                 msg.str(),
 186                 expected, actual);
 187 }
 188
 189 }
 190
 191 void TokenTest::testReader() {
 192         stringstream ss;
 193         ss <<
 194                 "/* booleans */\n"
 195                 "true false yes no on off\n"
 196                 "\"true\" \"false\" \"yes\" \"no\" \"on\" \"off\"\n"
 197                 "1 0 -1\n"
 198                 "# identifiers\n"
 199                 "foo foo_bar vec.y\n"
 200                 "// numbers\n"
 201                 "0 1 +2 -3 4.5\n"
 202                 ".5 1.5 0.25 -1.75 0.625\n"
 203                 "0 1 -1 2.5\n"
 204                 // strings
 205                 "\"hello\" \"\" \"\\r\\n\\t\\\"\"\n"
 206                 // vectors
 207                 "[1,0] [ 0.707, 0.707 ] // vec2\n"
 208                 "[.577,.577 ,0.577] [ 1,-2,3] // vec3\n"
 209                 "[ 0, 0, 0, 1 ] [1,0,0,-1.0] // vec4\n"
 210                 "[640, 480] [3, 4, 5] [0, -10, 100, -1000] # ivecs\n"
 211                 "[ -0.945, 0, -0.326, 0] # quat\n"
 212                 ;
 213         TokenStreamReader in(ss);
 214
 215         // booleans
 216
 217         bool value_bool;
 218         in.ReadBoolean(value_bool);
 219         assert_read("reading boolean true", true, value_bool, in);
 220         in.ReadBoolean(value_bool);
 221         assert_read("reading boolean false", false, value_bool, in);
 222         in.ReadBoolean(value_bool);
 223         assert_read("reading boolean yes", true, value_bool, in);
 224         in.ReadBoolean(value_bool);
 225         assert_read("reading boolean no", false, value_bool, in);
 226         in.ReadBoolean(value_bool);
 227         assert_read("reading boolean on", true, value_bool, in);
 228         in.ReadBoolean(value_bool);
 229         assert_read("reading boolean off", false, value_bool, in);
 230
 231         in.ReadBoolean(value_bool);
 232         assert_read("reading boolean \"true\"", true, value_bool, in);
 233         in.ReadBoolean(value_bool);
 234         assert_read("reading boolean \"false\"", false, value_bool, in);
 235         in.ReadBoolean(value_bool);
 236         assert_read("reading boolean \"yes\"", true, value_bool, in);
 237         in.ReadBoolean(value_bool);
 238         assert_read("reading boolean \"no\"", false, value_bool, in);
 239         in.ReadBoolean(value_bool);
 240         assert_read("reading boolean \"on\"", true, value_bool, in);
 241         in.ReadBoolean(value_bool);
 242         assert_read("reading boolean \"off\"", false, value_bool, in);
 243
 244         in.ReadBoolean(value_bool);
 245         assert_read("reading boolean 1", true, value_bool, in);
 246         in.ReadBoolean(value_bool);
 247         assert_read("reading boolean 0", false, value_bool, in);
 248         in.ReadBoolean(value_bool);
 249         assert_read("reading boolean -1", true, value_bool, in);
 250
 251         // identifiers
 252
 253         string value_ident;
 254         in.ReadIdentifier(value_ident);
 255         assert_read<string>("reading identifier foo", "foo", value_ident, in);
 256         in.ReadIdentifier(value_ident);
 257         assert_read<string>("reading identifier foo_bar", "foo_bar", value_ident, in);
 258         in.ReadIdentifier(value_ident);
 259         assert_read<string>("reading identifier vec.y", "vec.y", value_ident, in);
 260
 261         // numbers
 262         int value_int;
 263         in.ReadNumber(value_int);
 264         assert_read("reading integer 0", 0, value_int, in);
 265         in.ReadNumber(value_int);
 266         assert_read("reading integer 1", 1, value_int, in);
 267         in.ReadNumber(value_int);
 268         assert_read("reading integer +2", 2, value_int, in);
 269         in.ReadNumber(value_int);
 270         assert_read("reading integer -3", -3, value_int, in);
 271         in.ReadNumber(value_int);
 272         assert_read("reading integer 4.5", 4, value_int, in);
 273
 274         float value_float;
 275         in.ReadNumber(value_float);
 276         assert_read("reading float .5", .5f, value_float, in);
 277         in.ReadNumber(value_float);
 278         assert_read("reading float 1.5", 1.5f, value_float, in);
 279         in.ReadNumber(value_float);
 280         assert_read("reading float 0.25", .25f, value_float, in);
 281         in.ReadNumber(value_float);
 282         assert_read("reading float -1.75", -1.75f, value_float, in);
 283         in.ReadNumber(value_float);
 284         assert_read("reading float 0.625", 0.625f, value_float, in);
 285
 286         unsigned long value_uint;
 287         in.ReadNumber(value_uint);
 288         assert_read("reading unsigned integer 0", 0ul, value_uint, in);
 289         in.ReadNumber(value_uint);
 290         assert_read("reading unsigned integer 1", 1ul, value_uint, in);
 291         in.ReadNumber(value_uint);
 292         assert_read("reading unsigned integer -1", -1ul, value_uint, in);
 293         in.ReadNumber(value_uint);
 294         assert_read("reading unsigned integer 2.5", 2ul, value_uint, in);
 295
 296         // strings
 297
 298         string value_string;
 299         in.ReadString(value_string);
 300         assert_read<string>(
 301                 "reading string \"hello\"",
 302                 "hello", value_string, in);
 303         in.ReadString(value_string);
 304         assert_read<string>(
 305                 "reading string \"\"",
 306                 "", value_string, in);
 307         in.ReadString(value_string);
 308         assert_read<string>(
 309                 "reading string \"\\r\\n\\t\\\"\"",
 310                 "\r\n\t\"", value_string, in);
 311
 312         // vectors
 313
 314         glm::vec2 value_vec2;
 315         in.ReadVec(value_vec2);
 316         assert_read(
 317                 "reading vector [1,0]",
 318                 glm::vec2(1, 0), value_vec2, in);
 319         in.ReadVec(value_vec2);
 320         assert_read(
 321                 "reading vector [ 0.707, 0.707 ]",
 322                 glm::vec2(.707, .707), value_vec2, in);
 323
 324         glm::vec3 value_vec3;
 325         in.ReadVec(value_vec3);
 326         assert_read(
 327                 "reading vector [.577,.577 ,0.577]",
 328                 glm::vec3(.577, .577, .577), value_vec3, in);
 329         in.ReadVec(value_vec3);
 330         assert_read(
 331                 "reading vector [ 1,-2,3]",
 332                 glm::vec3(1, -2, 3), value_vec3, in);
 333
 334         glm::vec4 value_vec4;
 335         in.ReadVec(value_vec4);
 336         assert_read(
 337                 "reading vector [ 0, 0, 0, 1 ]",
 338                 glm::vec4(0, 0, 0, 1), value_vec4, in);
 339         in.ReadVec(value_vec4);
 340         assert_read(
 341                 "reading vector [1,0,0,-1.0]",
 342                 glm::vec4(1, 0, 0, -1), value_vec4, in);
 343
 344         glm::ivec2 value_ivec2;
 345         in.ReadVec(value_ivec2);
 346         assert_read(
 347                 "reading integer vector [640, 480]",
 348                 glm::ivec2(640, 480), value_ivec2, in);
 349         glm::ivec3 value_ivec3;
 350         in.ReadVec(value_ivec3);
 351         assert_read(
 352                 "reading integer vector [3, 4, 5]",
 353                 glm::ivec3(3, 4, 5), value_ivec3, in);
 354         glm::ivec4 value_ivec4;
 355         in.ReadVec(value_ivec4);
 356         assert_read(
 357                 "reading integer vector [0, -10, 100, -1000]",
 358                 glm::ivec4(0, -10, 100, -1000), value_ivec4, in);
 359
 360         glm::quat value_quat;
 361         in.ReadQuat(value_quat);
 362         assert_read(
 363                 "reading quaternion [ -0.945, 0, -0.326, 0]",
 364                 glm::quat(-0.945, 0, -0.326, 0), value_quat, in);
 365         // TODO: comment at end of stream makes it think there's more?
 366         //CPPUNIT_ASSERT_MESSAGE("expected end of stream", !in.HasMore());
 367         // TODO: and it even works??
 368         //CPPUNIT_ASSERT_THROW_MESSAGE(
 369         //      "extracting token after EOS",
 370         //      in.Next(), std::runtime_error);
 371 }
 372
 373 void TokenTest::testReaderEmpty() {
 374         { // zero length stream
 375                 stringstream ss;
 376                 ss << "";
 377                 TokenStreamReader in(ss);
 378                 CPPUNIT_ASSERT_MESSAGE(
 379                         "empty stream shouldn't have tokens",
 380                         !in.HasMore());
 381         }
 382         { // stream consisting solely of comments
 383                 stringstream ss;
 384                 ss <<
 385                         "/*\n"
 386                         " * hello\n"
 387                         " */\n"
 388                         "#hello\n"
 389                         "// is there anybody out there\n"
 390                         ;
 391                 TokenStreamReader in(ss);
 392                 CPPUNIT_ASSERT_MESSAGE(
 393                         "comment stream shouldn't have tokens",
 394                         !in.HasMore());
 395         }
 396 }
 397
 398 void TokenTest::testReaderMalformed() {
 399         {
 400                 stringstream ss;
 401                 ss << "a";
 402                 TokenStreamReader in(ss);
 403                 CPPUNIT_ASSERT_THROW_MESSAGE(
 404                         "unexpected token type should throw",
 405                         in.GetInt(), std::runtime_error);
 406         }
 407         {
 408                 stringstream ss;
 409                 ss << ":";
 410                 TokenStreamReader in(ss);
 411                 CPPUNIT_ASSERT_THROW_MESSAGE(
 412                         "casting ':' to bool should throw",
 413                         in.GetBool(), std::runtime_error);
 414         }
 415         {
 416                 stringstream ss;
 417                 ss << "hello";
 418                 TokenStreamReader in(ss);
 419                 CPPUNIT_ASSERT_THROW_MESSAGE(
 420                         "casting \"hello\" to bool should throw",
 421                         in.GetBool(), std::runtime_error);
 422         }
 423 }
 424
 425
 426 void TokenTest::AssertStreamOutput(
 427         Token::Type t,
 428         string expected
 429 ) {
 430         stringstream conv;
 431         conv << t;
 432         CPPUNIT_ASSERT_EQUAL_MESSAGE(
 433                 "unexpected std::ostream << Token::Type result",
 434                 expected, conv.str());
 435 }
 436
 437 void TokenTest::AssertStreamOutput(
 438         const Token &t,
 439         string expected
 440 ) {
 441         stringstream conv;
 442         conv << t;
 443         CPPUNIT_ASSERT_EQUAL_MESSAGE(
 444                 "unexpected std::ostream << Token result",
 445                 expected, conv.str());
 446 }
 447
 448 void TokenTest::AssertHasMore(Tokenizer &in) {
 449         CPPUNIT_ASSERT_MESSAGE("unexpected end of stream", in.HasMore());
 450 }
 451
 452 void TokenTest::AssertToken(
 453         Token::Type expected_type,
 454         const Token &actual_token
 455 ) {
 456         AssertToken(expected_type, "", actual_token);
 457 }
 458
 459 void TokenTest::AssertToken(
 460         Token::Type expected_type,
 461         string expected_value,
 462         const Token &actual_token
 463 ) {
 464         CPPUNIT_ASSERT_EQUAL_MESSAGE(
 465                 "unexpected token type",
 466                 expected_type, actual_token.type);
 467         CPPUNIT_ASSERT_EQUAL_MESSAGE(
 468                 "unexpected token value",
 469                 expected_value, actual_token.value);
 470 }
 471
 472 }
 473 }
 474 }