src/io/token.cpp

   1 #include "Tokenizer.hpp"
   2 #include "TokenStreamReader.hpp"
   3
   4 #include <cctype>
   5 #include <istream>
   6 #include <stdexcept>
   7 #include <glm/gtc/quaternion.hpp>
   8
   9 using namespace std;
  10
  11
  12 namespace blank {
  13
  14 Tokenizer::Tokenizer(istream &in)
  15 : in(in)
  16 , current() {
  17
  18 }
  19
  20
  21 bool Tokenizer::HasMore() {
  22         return bool(istream::sentry(in));
  23 }
  24
  25 const Token &Tokenizer::Next() {
  26         ReadToken();
  27         return Current();
  28 }
  29
  30 void Tokenizer::ReadToken() {
  31         current.type = Token::UNKNOWN;
  32         current.value.clear();
  33
  34         istream::sentry s(in);
  35         if (!s) {
  36                 throw runtime_error("read past the end of stream");
  37                 return;
  38         }
  39
  40         istream::char_type c;
  41         in.get(c);
  42         switch (c) {
  43                 case '{': case '}':
  44                 case '<': case '>':
  45                 case '[': case ']':
  46                 case '(': case ')':
  47                 case ';': case ':':
  48                 case ',': case '=':
  49                         current.type = Token::Type(c);
  50                         break;
  51                 case '+': case '-': case '.':
  52                 case '0': case '1': case '2': case '3': case '4':
  53                 case '5': case '6': case '7': case '8': case '9':
  54                         in.putback(c);
  55                         ReadNumber();
  56                         break;
  57                 case '"':
  58                         ReadString();
  59                         break;
  60                 case '#':
  61                 case '/':
  62                         in.putback(c);
  63                         ReadComment();
  64                         break;
  65                 default:
  66                         in.putback(c);
  67                         ReadIdentifier();
  68                         break;
  69         }
  70 }
  71
  72 namespace {
  73
  74 bool is_num_char(istream::char_type c) {
  75         return isxdigit(c)
  76                 || c == '.'
  77                 || c == '-'
  78                 || c == '+'
  79         ;
  80 }
  81
  82 }
  83
  84 void Tokenizer::ReadNumber() {
  85         current.type = Token::NUMBER;
  86         istream::char_type c;
  87         while (in.get(c)) {
  88                 if (is_num_char(c)) {
  89                         current.value += c;
  90                 } else {
  91                         in.putback(c);
  92                         break;
  93                 }
  94         }
  95 }
  96
  97 void Tokenizer::ReadString() {
  98         current.type = Token::STRING;
  99         bool escape = false;
 100
 101         istream::char_type c;
 102         while (in.get(c)) {
 103                 if (escape) {
 104                         escape = false;
 105                         switch (c) {
 106                                 case 'n':
 107                                         current.value += '\n';
 108                                         break;
 109                                 case 'r':
 110                                         current.value += '\t';
 111                                         break;
 112                                 case 't':
 113                                         current.value += '\t';
 114                                         break;
 115                                 default:
 116                                         current.value += c;
 117                                         break;
 118                         }
 119                 } else if (c == '"') {
 120                         break;
 121                 } else if (c == '\\') {
 122                         escape = true;
 123                 } else {
 124                         current.value += c;
 125                 }
 126         }
 127 }
 128
 129 void Tokenizer::ReadComment() {
 130         current.type = Token::COMMENT;
 131         istream::char_type c;
 132         in.get(c);
 133
 134         if (c == '#') {
 135                 while (in.get(c) && c != '\n') {
 136                         current.value += c;
 137                 }
 138                 return;
 139         }
 140
 141         // c is guaranteed to be '/' now
 142         if (!in.get(c)) {
 143                 throw runtime_error("unexpected end of stream");
 144         }
 145         if (c == '/') {
 146                 while (in.get(c) && c != '\n') {
 147                         current.value += c;
 148                 }
 149                 return;
 150         } else if (c != '*') {
 151                 throw runtime_error("invalid character after /");
 152         }
 153
 154         while (in.get(c)) {
 155                 if (c == '*') {
 156                         istream::char_type c2;
 157                         if (!in.get(c2)) {
 158                                 throw runtime_error("unexpected end of stream");
 159                         }
 160                         if (c2 == '/') {
 161                                 break;
 162                         } else {
 163                                 current.value += c;
 164                                 current.value += c2;
 165                         }
 166                 } else {
 167                         current.value += c;
 168                 }
 169         }
 170 }
 171
 172 void Tokenizer::ReadIdentifier() {
 173         current.type = Token::IDENTIFIER;
 174
 175         istream::char_type c;
 176         while (in.get(c)) {
 177                 if (isalnum(c) || c == '_') {
 178                         current.value += c;
 179                 } else {
 180                         in.putback(c);
 181                         break;
 182                 }
 183         }
 184 }
 185
 186
 187 TokenStreamReader::TokenStreamReader(istream &in)
 188 : in(in)
 189 , cached(false) {
 190
 191 }
 192
 193
 194 bool TokenStreamReader::HasMore() {
 195         if (cached) {
 196                 return true;
 197         }
 198         while (in.HasMore()) {
 199                 if (in.Next().type != Token::COMMENT) {
 200                         cached = true;
 201                         return true;
 202                 }
 203         }
 204         return false;
 205 }
 206
 207 const Token &TokenStreamReader::Next() {
 208         if (cached) {
 209                 cached = false;
 210                 return in.Current();
 211         } else {
 212                 return in.Next();
 213         }
 214 }
 215
 216 const Token &TokenStreamReader::Peek() {
 217         if (!cached) {
 218                 in.Next();
 219                 cached = true;
 220         }
 221         return in.Current();
 222 }
 223
 224
 225 void TokenStreamReader::Assert(Token::Type t) {
 226         if (GetType() != t) {
 227                 throw runtime_error("unexpected token in input stream");
 228         }
 229 }
 230
 231 Token::Type TokenStreamReader::GetType() const noexcept {
 232         return in.Current().type;
 233 }
 234
 235 const std::string &TokenStreamReader::GetValue() const noexcept {
 236         return in.Current().value;
 237 }
 238
 239 void TokenStreamReader::Skip(Token::Type t) {
 240         Next();
 241         Assert(t);
 242 }
 243
 244
 245 void TokenStreamReader::ReadBoolean(bool &b) {
 246         b = GetBool();
 247 }
 248
 249 void TokenStreamReader::ReadIdentifier(string &out) {
 250         Next();
 251         Assert(Token::IDENTIFIER);
 252         out = GetValue();
 253 }
 254
 255 void TokenStreamReader::ReadNumber(float &n) {
 256         n = GetFloat();
 257 }
 258
 259 void TokenStreamReader::ReadNumber(int &n) {
 260         n = GetInt();
 261 }
 262
 263 void TokenStreamReader::ReadNumber(unsigned long &n) {
 264         n = GetULong();
 265 }
 266
 267 void TokenStreamReader::ReadString(string &out) {
 268         Next();
 269         Assert(Token::STRING);
 270         out = GetValue();
 271 }
 272
 273
 274 void TokenStreamReader::ReadVec(glm::vec2 &v) {
 275         Skip(Token::BRACKET_OPEN);
 276         ReadNumber(v.x);
 277         Skip(Token::COMMA);
 278         ReadNumber(v.y);
 279         Skip(Token::BRACKET_CLOSE);
 280 }
 281
 282 void TokenStreamReader::ReadVec(glm::vec3 &v) {
 283         Skip(Token::BRACKET_OPEN);
 284         ReadNumber(v.x);
 285         Skip(Token::COMMA);
 286         ReadNumber(v.y);
 287         Skip(Token::COMMA);
 288         ReadNumber(v.z);
 289         Skip(Token::BRACKET_CLOSE);
 290 }
 291
 292 void TokenStreamReader::ReadVec(glm::vec4 &v) {
 293         Skip(Token::BRACKET_OPEN);
 294         ReadNumber(v.x);
 295         Skip(Token::COMMA);
 296         ReadNumber(v.y);
 297         Skip(Token::COMMA);
 298         ReadNumber(v.z);
 299         Skip(Token::COMMA);
 300         ReadNumber(v.w);
 301         Skip(Token::BRACKET_CLOSE);
 302 }
 303
 304 void TokenStreamReader::ReadVec(glm::ivec2 &v) {
 305         Skip(Token::BRACKET_OPEN);
 306         ReadNumber(v.x);
 307         Skip(Token::COMMA);
 308         ReadNumber(v.y);
 309         Skip(Token::BRACKET_CLOSE);
 310 }
 311
 312 void TokenStreamReader::ReadVec(glm::ivec3 &v) {
 313         Skip(Token::BRACKET_OPEN);
 314         ReadNumber(v.x);
 315         Skip(Token::COMMA);
 316         ReadNumber(v.y);
 317         Skip(Token::COMMA);
 318         ReadNumber(v.z);
 319         Skip(Token::BRACKET_CLOSE);
 320 }
 321
 322 void TokenStreamReader::ReadVec(glm::ivec4 &v) {
 323         Skip(Token::BRACKET_OPEN);
 324         ReadNumber(v.x);
 325         Skip(Token::COMMA);
 326         ReadNumber(v.y);
 327         Skip(Token::COMMA);
 328         ReadNumber(v.z);
 329         Skip(Token::COMMA);
 330         ReadNumber(v.w);
 331         Skip(Token::BRACKET_CLOSE);
 332 }
 333
 334 void TokenStreamReader::ReadQuat(glm::quat &q) {
 335         Skip(Token::BRACKET_OPEN);
 336         ReadNumber(q.w);
 337         Skip(Token::COMMA);
 338         ReadNumber(q.x);
 339         Skip(Token::COMMA);
 340         ReadNumber(q.y);
 341         Skip(Token::COMMA);
 342         ReadNumber(q.z);
 343         Skip(Token::BRACKET_CLOSE);
 344 }
 345
 346
 347 bool TokenStreamReader::GetBool() {
 348         Next();
 349         switch (GetType()) {
 350                 case Token::NUMBER:
 351                         return GetInt() != 0;
 352                 case Token::IDENTIFIER:
 353                 case Token::STRING:
 354                         if (GetValue() == "true" || GetValue() == "yes" || GetValue() == "on") {
 355                                 return true;
 356                         } else if (GetValue() == "false" || GetValue() == "no" || GetValue() == "off") {
 357                                 return false;
 358                         } else {
 359                                 throw runtime_error("unexpected value in input stream");
 360                         }
 361                 default:
 362                         throw runtime_error("unexpected token in input stream");
 363         }
 364 }
 365
 366 float TokenStreamReader::GetFloat() {
 367         Next();
 368         Assert(Token::NUMBER);
 369         return stof(GetValue());
 370 }
 371
 372 int TokenStreamReader::GetInt() {
 373         Next();
 374         Assert(Token::NUMBER);
 375         return stoi(GetValue());
 376 }
 377
 378 unsigned long TokenStreamReader::GetULong() {
 379         Next();
 380         Assert(Token::NUMBER);
 381         return stoul(GetValue());
 382 }
 383
 384 }