src/io/token.cpp

   1 #include "Token.hpp"
   2 #include "Tokenizer.hpp"
   3 #include "TokenStreamReader.hpp"
   4
   5 #include <cctype>
   6 #include <istream>
   7 #include <ostream>
   8 #include <sstream>
   9 #include <stdexcept>
  10 #include <glm/gtc/quaternion.hpp>
  11
  12 using namespace std;
  13
  14
  15 namespace gong {
  16 namespace io {
  17
  18 ostream &operator <<(ostream &out, Token::Type t) {
  19         switch (t) {
  20                 case Token::ANGLE_BRACKET_OPEN:
  21                         return out << "ANGLE_BRACKET_OPEN";
  22                 case Token::ANGLE_BRACKET_CLOSE:
  23                         return out << "ANGLE_BRACKET_CLOSE";
  24                 case Token::CHEVRON_OPEN:
  25                         return out << "CHEVRON_OPEN";
  26                 case Token::CHEVRON_CLOSE:
  27                         return out << "CHEVRON_CLOSE";
  28                 case Token::BRACKET_OPEN:
  29                         return out << "BRACKET_OPEN";
  30                 case Token::BRACKET_CLOSE:
  31                         return out << "BRACKET_CLOSE";
  32                 case Token::PARENTHESIS_OPEN:
  33                         return out << "PARENTHESIS_OPEN";
  34                 case Token::PARENTHESIS_CLOSE:
  35                         return out << "PARENTHESIS_CLOSE";
  36                 case Token::COLON:
  37                         return out << "COLON";
  38                 case Token::SEMICOLON:
  39                         return out << "SEMICOLON";
  40                 case Token::COMMA:
  41                         return out << "COMMA";
  42                 case Token::EQUALS:
  43                         return out << "EQUALS";
  44                 case Token::NUMBER:
  45                         return out << "NUMBER";
  46                 case Token::STRING:
  47                         return out << "STRING";
  48                 case Token::IDENTIFIER:
  49                         return out << "IDENTIFIER";
  50                 case Token::COMMENT:
  51                         return out << "COMMENT";
  52                 default:
  53                         return out << "UNKNOWN";
  54         }
  55 }
  56
  57 ostream &operator <<(ostream &out, const Token &t) {
  58         out << t.type;
  59         switch (t.type) {
  60                 case Token::UNKNOWN:
  61                 case Token::NUMBER:
  62                 case Token::STRING:
  63                 case Token::IDENTIFIER:
  64                 case Token::COMMENT:
  65                         return out << '(' << t.value << ')';
  66                 default:
  67                         return out;
  68         }
  69 }
  70
  71 Tokenizer::Tokenizer(istream &in)
  72 : in(in)
  73 , current() {
  74
  75 }
  76
  77
  78 bool Tokenizer::HasMore() {
  79         return bool(istream::sentry(in));
  80 }
  81
  82 const Token &Tokenizer::Next() {
  83         ReadToken();
  84         return Current();
  85 }
  86
  87 void Tokenizer::ReadToken() {
  88         current.type = Token::UNKNOWN;
  89         current.value.clear();
  90
  91         istream::sentry s(in);
  92         if (!s) {
  93                 throw runtime_error("read past the end of stream");
  94                 return;
  95         }
  96
  97         istream::char_type c;
  98         in.get(c);
  99         switch (c) {
 100                 case '{': case '}':
 101                 case '<': case '>':
 102                 case '[': case ']':
 103                 case '(': case ')':
 104                 case ';': case ':':
 105                 case ',': case '=':
 106                         current.type = Token::Type(c);
 107                         break;
 108                 case '+': case '-': case '.':
 109                 case '0': case '1': case '2': case '3': case '4':
 110                 case '5': case '6': case '7': case '8': case '9':
 111                         in.putback(c);
 112                         ReadNumber();
 113                         break;
 114                 case '"':
 115                         ReadString();
 116                         break;
 117                 case '#':
 118                 case '/':
 119                         in.putback(c);
 120                         ReadComment();
 121                         break;
 122                 default:
 123                         in.putback(c);
 124                         ReadIdentifier();
 125                         break;
 126         }
 127 }
 128
 129 namespace {
 130
 131 bool is_num_char(istream::char_type c) {
 132         return isxdigit(c)
 133                 || c == '.'
 134                 || c == '-'
 135                 || c == '+'
 136         ;
 137 }
 138
 139 }
 140
 141 void Tokenizer::ReadNumber() {
 142         current.type = Token::NUMBER;
 143         istream::char_type c;
 144         while (in.get(c)) {
 145                 if (is_num_char(c)) {
 146                         current.value += c;
 147                 } else {
 148                         in.putback(c);
 149                         break;
 150                 }
 151         }
 152 }
 153
 154 void Tokenizer::ReadString() {
 155         current.type = Token::STRING;
 156         bool escape = false;
 157
 158         istream::char_type c;
 159         while (in.get(c)) {
 160                 if (escape) {
 161                         escape = false;
 162                         switch (c) {
 163                                 case 'n':
 164                                         current.value += '\n';
 165                                         break;
 166                                 case 'r':
 167                                         current.value += '\r';
 168                                         break;
 169                                 case 't':
 170                                         current.value += '\t';
 171                                         break;
 172                                 default:
 173                                         current.value += c;
 174                                         break;
 175                         }
 176                 } else if (c == '"') {
 177                         break;
 178                 } else if (c == '\\') {
 179                         escape = true;
 180                 } else {
 181                         current.value += c;
 182                 }
 183         }
 184 }
 185
 186 void Tokenizer::ReadComment() {
 187         current.type = Token::COMMENT;
 188         istream::char_type c;
 189         in.get(c);
 190
 191         if (c == '#') {
 192                 while (in.get(c) && c != '\n') {
 193                         current.value += c;
 194                 }
 195                 return;
 196         }
 197
 198         // c is guaranteed to be '/' now
 199         if (!in.get(c)) {
 200                 throw runtime_error("unexpected end of stream");
 201         }
 202         if (c == '/') {
 203                 while (in.get(c) && c != '\n') {
 204                         current.value += c;
 205                 }
 206                 return;
 207         } else if (c != '*') {
 208                 throw runtime_error("invalid character after /");
 209         }
 210
 211         while (in.get(c)) {
 212                 if (c == '*') {
 213                         istream::char_type c2;
 214                         if (!in.get(c2)) {
 215                                 throw runtime_error("unexpected end of stream");
 216                         }
 217                         if (c2 == '/') {
 218                                 break;
 219                         } else {
 220                                 current.value += c;
 221                                 current.value += c2;
 222                         }
 223                 } else {
 224                         current.value += c;
 225                 }
 226         }
 227 }
 228
 229 void Tokenizer::ReadIdentifier() {
 230         current.type = Token::IDENTIFIER;
 231
 232         istream::char_type c;
 233         while (in.get(c)) {
 234                 if (isalnum(c) || c == '_' || c == '.') {
 235                         current.value += c;
 236                 } else {
 237                         in.putback(c);
 238                         break;
 239                 }
 240         }
 241 }
 242
 243
 244 TokenStreamReader::TokenStreamReader(istream &in)
 245 : in(in)
 246 , cached(false) {
 247
 248 }
 249
 250
 251 bool TokenStreamReader::HasMore() {
 252         if (cached) {
 253                 return true;
 254         }
 255         SkipComments();
 256         return cached;
 257 }
 258
 259 const Token &TokenStreamReader::Next() {
 260         SkipComments();
 261         cached = false;
 262         return in.Current();
 263 }
 264
 265 void TokenStreamReader::SkipComments() {
 266         if (cached) {
 267                 if (in.Current().type == Token::COMMENT) {
 268                         cached = false;
 269                 } else {
 270                         return;
 271                 }
 272         }
 273         while (in.HasMore()) {
 274                 if (in.Next().type != Token::COMMENT) {
 275                         cached = true;
 276                         return;
 277                 }
 278         }
 279 }
 280
 281 const Token &TokenStreamReader::Peek() {
 282         if (!cached) {
 283                 Next();
 284                 cached = true;
 285         }
 286         return in.Current();
 287 }
 288
 289
 290 void TokenStreamReader::Assert(Token::Type t) const {
 291         if (GetType() != t) {
 292                 stringstream s;
 293                 s << "unexpected token in input stream: expected " << t << ", but got " << in.Current();
 294                 throw runtime_error(s.str());
 295         }
 296 }
 297
 298 Token::Type TokenStreamReader::GetType() const noexcept {
 299         return in.Current().type;
 300 }
 301
 302 const std::string &TokenStreamReader::GetValue() const noexcept {
 303         return in.Current().value;
 304 }
 305
 306 void TokenStreamReader::Skip(Token::Type t) {
 307         Next();
 308         Assert(t);
 309 }
 310
 311
 312 void TokenStreamReader::ReadBoolean(bool &b) {
 313         b = GetBool();
 314 }
 315
 316 void TokenStreamReader::ReadIdentifier(string &out) {
 317         Next();
 318         Assert(Token::IDENTIFIER);
 319         out = GetValue();
 320 }
 321
 322 void TokenStreamReader::ReadNumber(float &n) {
 323         n = GetFloat();
 324 }
 325
 326 void TokenStreamReader::ReadNumber(int &n) {
 327         n = GetInt();
 328 }
 329
 330 void TokenStreamReader::ReadNumber(unsigned long &n) {
 331         n = GetULong();
 332 }
 333
 334 void TokenStreamReader::ReadString(string &out) {
 335         Next();
 336         Assert(Token::STRING);
 337         out = GetValue();
 338 }
 339
 340 void TokenStreamReader::ReadRelaxedString(string &out) {
 341         out = GetString();
 342 }
 343
 344
 345 void TokenStreamReader::ReadVec(glm::vec2 &v) {
 346         Skip(Token::BRACKET_OPEN);
 347         ReadNumber(v.x);
 348         Skip(Token::COMMA);
 349         ReadNumber(v.y);
 350         Skip(Token::BRACKET_CLOSE);
 351 }
 352
 353 void TokenStreamReader::ReadVec(glm::vec3 &v) {
 354         Skip(Token::BRACKET_OPEN);
 355         ReadNumber(v.x);
 356         Skip(Token::COMMA);
 357         ReadNumber(v.y);
 358         Skip(Token::COMMA);
 359         ReadNumber(v.z);
 360         Skip(Token::BRACKET_CLOSE);
 361 }
 362
 363 void TokenStreamReader::ReadVec(glm::vec4 &v) {
 364         Skip(Token::BRACKET_OPEN);
 365         ReadNumber(v.x);
 366         Skip(Token::COMMA);
 367         ReadNumber(v.y);
 368         Skip(Token::COMMA);
 369         ReadNumber(v.z);
 370         Skip(Token::COMMA);
 371         ReadNumber(v.w);
 372         Skip(Token::BRACKET_CLOSE);
 373 }
 374
 375 void TokenStreamReader::ReadVec(glm::ivec2 &v) {
 376         Skip(Token::BRACKET_OPEN);
 377         ReadNumber(v.x);
 378         Skip(Token::COMMA);
 379         ReadNumber(v.y);
 380         Skip(Token::BRACKET_CLOSE);
 381 }
 382
 383 void TokenStreamReader::ReadVec(glm::ivec3 &v) {
 384         Skip(Token::BRACKET_OPEN);
 385         ReadNumber(v.x);
 386         Skip(Token::COMMA);
 387         ReadNumber(v.y);
 388         Skip(Token::COMMA);
 389         ReadNumber(v.z);
 390         Skip(Token::BRACKET_CLOSE);
 391 }
 392
 393 void TokenStreamReader::ReadVec(glm::ivec4 &v) {
 394         Skip(Token::BRACKET_OPEN);
 395         ReadNumber(v.x);
 396         Skip(Token::COMMA);
 397         ReadNumber(v.y);
 398         Skip(Token::COMMA);
 399         ReadNumber(v.z);
 400         Skip(Token::COMMA);
 401         ReadNumber(v.w);
 402         Skip(Token::BRACKET_CLOSE);
 403 }
 404
 405 void TokenStreamReader::ReadQuat(glm::quat &q) {
 406         Skip(Token::BRACKET_OPEN);
 407         ReadNumber(q.w);
 408         Skip(Token::COMMA);
 409         ReadNumber(q.x);
 410         Skip(Token::COMMA);
 411         ReadNumber(q.y);
 412         Skip(Token::COMMA);
 413         ReadNumber(q.z);
 414         Skip(Token::BRACKET_CLOSE);
 415 }
 416
 417
 418 bool TokenStreamReader::GetBool() {
 419         Next();
 420         return AsBool();
 421 }
 422
 423 bool TokenStreamReader::AsBool() const {
 424         switch (GetType()) {
 425                 case Token::NUMBER:
 426                         return AsInt() != 0;
 427                 case Token::IDENTIFIER:
 428                 case Token::STRING:
 429                         if (GetValue() == "true" || GetValue() == "yes" || GetValue() == "on") {
 430                                 return true;
 431                         } else if (GetValue() == "false" || GetValue() == "no" || GetValue() == "off") {
 432                                 return false;
 433                         } else {
 434                                 throw runtime_error("unexpected value in input stream: cannot cast " + GetValue() + " to bool");
 435                         }
 436                 default:
 437                         {
 438                                 stringstream s;
 439                                 s << "unexpected token in input stream: cannot cast " << in.Current() << " to bool";
 440                                 throw runtime_error(s.str());
 441                         }
 442         }
 443 }
 444
 445 float TokenStreamReader::GetFloat() {
 446         Next();
 447         return AsFloat();
 448 }
 449
 450 float TokenStreamReader::AsFloat() const {
 451         Assert(Token::NUMBER);
 452         return stof(GetValue());
 453 }
 454
 455 int TokenStreamReader::GetInt() {
 456         Next();
 457         return AsInt();
 458 }
 459
 460 int TokenStreamReader::AsInt() const {
 461         Assert(Token::NUMBER);
 462         return stoi(GetValue());
 463 }
 464
 465 unsigned long TokenStreamReader::GetULong() {
 466         Next();
 467         return AsULong();
 468 }
 469
 470 unsigned long TokenStreamReader::AsULong() const {
 471         Assert(Token::NUMBER);
 472         return stoul(GetValue());
 473 }
 474
 475 const string &TokenStreamReader::GetString() {
 476         Next();
 477         return AsString();
 478 }
 479
 480 const string &TokenStreamReader::AsString() const {
 481         return GetValue();
 482 }
 483
 484 }
 485 }