src/io/token.cpp

   1 #include "Token.hpp"
   2 #include "Tokenizer.hpp"
   3 #include "TokenStreamReader.hpp"
   4
   5 #include <cctype>
   6 #include <istream>
   7 #include <ostream>
   8 #include <sstream>
   9 #include <stdexcept>
  10 #include <glm/gtc/quaternion.hpp>
  11
  12 using namespace std;
  13
  14
  15 namespace blank {
  16
  17 ostream &operator <<(ostream &out, Token::Type t) {
  18         switch (t) {
  19                 case Token::ANGLE_BRACKET_OPEN:
  20                         return out << "ANGLE_BRACKET_OPEN";
  21                 case Token::ANGLE_BRACKET_CLOSE:
  22                         return out << "ANGLE_BRACKET_CLOSE";
  23                 case Token::CHEVRON_OPEN:
  24                         return out << "CHEVRON_OPEN";
  25                 case Token::CHEVRON_CLOSE:
  26                         return out << "CHEVRON_CLOSE";
  27                 case Token::BRACKET_OPEN:
  28                         return out << "BRACKET_OPEN";
  29                 case Token::BRACKET_CLOSE:
  30                         return out << "BRACKET_CLOSE";
  31                 case Token::PARENTHESIS_OPEN:
  32                         return out << "PARENTHESIS_OPEN";
  33                 case Token::PARENTHESIS_CLOSE:
  34                         return out << "PARENTHESIS_CLOSE";
  35                 case Token::COLON:
  36                         return out << "COLON";
  37                 case Token::SEMICOLON:
  38                         return out << "SEMICOLON";
  39                 case Token::COMMA:
  40                         return out << "COMMA";
  41                 case Token::EQUALS:
  42                         return out << "EQUALS";
  43                 case Token::NUMBER:
  44                         return out << "NUMBER";
  45                 case Token::STRING:
  46                         return out << "STRING";
  47                 case Token::IDENTIFIER:
  48                         return out << "IDENTIFIER";
  49                 case Token::COMMENT:
  50                         return out << "COMMENT";
  51                 default:
  52                         return out << "UNKNOWN";
  53         }
  54 }
  55
  56 ostream &operator <<(ostream &out, const Token &t) {
  57         out << t.type;
  58         switch (t.type) {
  59                 case Token::UNKNOWN:
  60                 case Token::NUMBER:
  61                 case Token::STRING:
  62                 case Token::IDENTIFIER:
  63                 case Token::COMMENT:
  64                         return out << '(' << t.value << ')';
  65                 default:
  66                         return out;
  67         }
  68 }
  69
  70 Tokenizer::Tokenizer(istream &in)
  71 : in(in)
  72 , current() {
  73
  74 }
  75
  76
  77 bool Tokenizer::HasMore() {
  78         return bool(istream::sentry(in));
  79 }
  80
  81 const Token &Tokenizer::Next() {
  82         ReadToken();
  83         return Current();
  84 }
  85
  86 void Tokenizer::ReadToken() {
  87         current.type = Token::UNKNOWN;
  88         current.value.clear();
  89
  90         istream::sentry s(in);
  91         if (!s) {
  92                 throw runtime_error("read past the end of stream");
  93                 return;
  94         }
  95
  96         istream::char_type c;
  97         in.get(c);
  98         switch (c) {
  99                 case '{': case '}':
 100                 case '<': case '>':
 101                 case '[': case ']':
 102                 case '(': case ')':
 103                 case ';': case ':':
 104                 case ',': case '=':
 105                         current.type = Token::Type(c);
 106                         break;
 107                 case '+': case '-': case '.':
 108                 case '0': case '1': case '2': case '3': case '4':
 109                 case '5': case '6': case '7': case '8': case '9':
 110                         in.putback(c);
 111                         ReadNumber();
 112                         break;
 113                 case '"':
 114                         ReadString();
 115                         break;
 116                 case '#':
 117                 case '/':
 118                         in.putback(c);
 119                         ReadComment();
 120                         break;
 121                 default:
 122                         in.putback(c);
 123                         ReadIdentifier();
 124                         break;
 125         }
 126 }
 127
 128 namespace {
 129
 130 bool is_num_char(istream::char_type c) {
 131         return isxdigit(c)
 132                 || c == '.'
 133                 || c == '-'
 134                 || c == '+'
 135         ;
 136 }
 137
 138 }
 139
 140 void Tokenizer::ReadNumber() {
 141         current.type = Token::NUMBER;
 142         istream::char_type c;
 143         while (in.get(c)) {
 144                 if (is_num_char(c)) {
 145                         current.value += c;
 146                 } else {
 147                         in.putback(c);
 148                         break;
 149                 }
 150         }
 151 }
 152
 153 void Tokenizer::ReadString() {
 154         current.type = Token::STRING;
 155         bool escape = false;
 156
 157         istream::char_type c;
 158         while (in.get(c)) {
 159                 if (escape) {
 160                         escape = false;
 161                         switch (c) {
 162                                 case 'n':
 163                                         current.value += '\n';
 164                                         break;
 165                                 case 'r':
 166                                         current.value += '\t';
 167                                         break;
 168                                 case 't':
 169                                         current.value += '\t';
 170                                         break;
 171                                 default:
 172                                         current.value += c;
 173                                         break;
 174                         }
 175                 } else if (c == '"') {
 176                         break;
 177                 } else if (c == '\\') {
 178                         escape = true;
 179                 } else {
 180                         current.value += c;
 181                 }
 182         }
 183 }
 184
 185 void Tokenizer::ReadComment() {
 186         current.type = Token::COMMENT;
 187         istream::char_type c;
 188         in.get(c);
 189
 190         if (c == '#') {
 191                 while (in.get(c) && c != '\n') {
 192                         current.value += c;
 193                 }
 194                 return;
 195         }
 196
 197         // c is guaranteed to be '/' now
 198         if (!in.get(c)) {
 199                 throw runtime_error("unexpected end of stream");
 200         }
 201         if (c == '/') {
 202                 while (in.get(c) && c != '\n') {
 203                         current.value += c;
 204                 }
 205                 return;
 206         } else if (c != '*') {
 207                 throw runtime_error("invalid character after /");
 208         }
 209
 210         while (in.get(c)) {
 211                 if (c == '*') {
 212                         istream::char_type c2;
 213                         if (!in.get(c2)) {
 214                                 throw runtime_error("unexpected end of stream");
 215                         }
 216                         if (c2 == '/') {
 217                                 break;
 218                         } else {
 219                                 current.value += c;
 220                                 current.value += c2;
 221                         }
 222                 } else {
 223                         current.value += c;
 224                 }
 225         }
 226 }
 227
 228 void Tokenizer::ReadIdentifier() {
 229         current.type = Token::IDENTIFIER;
 230
 231         istream::char_type c;
 232         while (in.get(c)) {
 233                 if (isalnum(c) || c == '_' || c == '.') {
 234                         current.value += c;
 235                 } else {
 236                         in.putback(c);
 237                         break;
 238                 }
 239         }
 240 }
 241
 242
 243 TokenStreamReader::TokenStreamReader(istream &in)
 244 : in(in)
 245 , cached(false) {
 246
 247 }
 248
 249
 250 bool TokenStreamReader::HasMore() {
 251         if (cached) {
 252                 return true;
 253         }
 254         SkipComments();
 255         return cached;
 256 }
 257
 258 const Token &TokenStreamReader::Next() {
 259         SkipComments();
 260         cached = false;
 261         return in.Current();
 262 }
 263
 264 void TokenStreamReader::SkipComments() {
 265         if (cached) {
 266                 if (in.Current().type == Token::COMMENT) {
 267                         cached = false;
 268                 } else {
 269                         return;
 270                 }
 271         }
 272         while (in.HasMore()) {
 273                 if (in.Next().type != Token::COMMENT) {
 274                         cached = true;
 275                         return;
 276                 }
 277         }
 278 }
 279
 280 const Token &TokenStreamReader::Peek() {
 281         if (!cached) {
 282                 Next();
 283                 cached = true;
 284         }
 285         return in.Current();
 286 }
 287
 288
 289 void TokenStreamReader::Assert(Token::Type t) {
 290         if (GetType() != t) {
 291                 stringstream s;
 292                 s << "unexpected token in input stream: expected " << t << ", but got " << in.Current();
 293                 throw runtime_error(s.str());
 294         }
 295 }
 296
 297 Token::Type TokenStreamReader::GetType() const noexcept {
 298         return in.Current().type;
 299 }
 300
 301 const std::string &TokenStreamReader::GetValue() const noexcept {
 302         return in.Current().value;
 303 }
 304
 305 void TokenStreamReader::Skip(Token::Type t) {
 306         Next();
 307         Assert(t);
 308 }
 309
 310
 311 void TokenStreamReader::ReadBoolean(bool &b) {
 312         b = GetBool();
 313 }
 314
 315 void TokenStreamReader::ReadIdentifier(string &out) {
 316         Next();
 317         Assert(Token::IDENTIFIER);
 318         out = GetValue();
 319 }
 320
 321 void TokenStreamReader::ReadNumber(float &n) {
 322         n = GetFloat();
 323 }
 324
 325 void TokenStreamReader::ReadNumber(int &n) {
 326         n = GetInt();
 327 }
 328
 329 void TokenStreamReader::ReadNumber(unsigned long &n) {
 330         n = GetULong();
 331 }
 332
 333 void TokenStreamReader::ReadString(string &out) {
 334         Next();
 335         Assert(Token::STRING);
 336         out = GetValue();
 337 }
 338
 339
 340 void TokenStreamReader::ReadVec(glm::vec2 &v) {
 341         Skip(Token::BRACKET_OPEN);
 342         ReadNumber(v.x);
 343         Skip(Token::COMMA);
 344         ReadNumber(v.y);
 345         Skip(Token::BRACKET_CLOSE);
 346 }
 347
 348 void TokenStreamReader::ReadVec(glm::vec3 &v) {
 349         Skip(Token::BRACKET_OPEN);
 350         ReadNumber(v.x);
 351         Skip(Token::COMMA);
 352         ReadNumber(v.y);
 353         Skip(Token::COMMA);
 354         ReadNumber(v.z);
 355         Skip(Token::BRACKET_CLOSE);
 356 }
 357
 358 void TokenStreamReader::ReadVec(glm::vec4 &v) {
 359         Skip(Token::BRACKET_OPEN);
 360         ReadNumber(v.x);
 361         Skip(Token::COMMA);
 362         ReadNumber(v.y);
 363         Skip(Token::COMMA);
 364         ReadNumber(v.z);
 365         Skip(Token::COMMA);
 366         ReadNumber(v.w);
 367         Skip(Token::BRACKET_CLOSE);
 368 }
 369
 370 void TokenStreamReader::ReadVec(glm::ivec2 &v) {
 371         Skip(Token::BRACKET_OPEN);
 372         ReadNumber(v.x);
 373         Skip(Token::COMMA);
 374         ReadNumber(v.y);
 375         Skip(Token::BRACKET_CLOSE);
 376 }
 377
 378 void TokenStreamReader::ReadVec(glm::ivec3 &v) {
 379         Skip(Token::BRACKET_OPEN);
 380         ReadNumber(v.x);
 381         Skip(Token::COMMA);
 382         ReadNumber(v.y);
 383         Skip(Token::COMMA);
 384         ReadNumber(v.z);
 385         Skip(Token::BRACKET_CLOSE);
 386 }
 387
 388 void TokenStreamReader::ReadVec(glm::ivec4 &v) {
 389         Skip(Token::BRACKET_OPEN);
 390         ReadNumber(v.x);
 391         Skip(Token::COMMA);
 392         ReadNumber(v.y);
 393         Skip(Token::COMMA);
 394         ReadNumber(v.z);
 395         Skip(Token::COMMA);
 396         ReadNumber(v.w);
 397         Skip(Token::BRACKET_CLOSE);
 398 }
 399
 400 void TokenStreamReader::ReadQuat(glm::quat &q) {
 401         Skip(Token::BRACKET_OPEN);
 402         ReadNumber(q.w);
 403         Skip(Token::COMMA);
 404         ReadNumber(q.x);
 405         Skip(Token::COMMA);
 406         ReadNumber(q.y);
 407         Skip(Token::COMMA);
 408         ReadNumber(q.z);
 409         Skip(Token::BRACKET_CLOSE);
 410 }
 411
 412
 413 bool TokenStreamReader::GetBool() {
 414         Next();
 415         switch (GetType()) {
 416                 case Token::NUMBER:
 417                         return GetInt() != 0;
 418                 case Token::IDENTIFIER:
 419                 case Token::STRING:
 420                         if (GetValue() == "true" || GetValue() == "yes" || GetValue() == "on") {
 421                                 return true;
 422                         } else if (GetValue() == "false" || GetValue() == "no" || GetValue() == "off") {
 423                                 return false;
 424                         } else {
 425                                 throw runtime_error("unexpected value in input stream: cannot cast " + GetValue() + " to bool");
 426                         }
 427                 default:
 428                         {
 429                                 stringstream s;
 430                                 s << "unexpected token in input stream: cannot cast " << in.Current() << " to bool";
 431                                 throw runtime_error(s.str());
 432                         }
 433         }
 434 }
 435
 436 float TokenStreamReader::GetFloat() {
 437         Next();
 438         Assert(Token::NUMBER);
 439         return stof(GetValue());
 440 }
 441
 442 int TokenStreamReader::GetInt() {
 443         Next();
 444         Assert(Token::NUMBER);
 445         return stoi(GetValue());
 446 }
 447
 448 unsigned long TokenStreamReader::GetULong() {
 449         Next();
 450         Assert(Token::NUMBER);
 451         return stoul(GetValue());
 452 }
 453
 454 }