src/io/token.cpp

   1 #include "Tokenizer.hpp"
   2 #include "TokenStreamReader.hpp"
   3
   4 #include <cctype>
   5 #include <istream>
   6 #include <stdexcept>
   7
   8 using namespace std;
   9
  10
  11 namespace blank {
  12
  13 Tokenizer::Tokenizer(istream &in)
  14 : in(in)
  15 , current() {
  16
  17 }
  18
  19
  20 bool Tokenizer::HasMore() {
  21         return bool(istream::sentry(in));
  22 }
  23
  24 const Token &Tokenizer::Next() {
  25         ReadToken();
  26         return Current();
  27 }
  28
  29 void Tokenizer::ReadToken() {
  30         current.type = Token::UNKNOWN;
  31         current.value.clear();
  32
  33         istream::sentry s(in);
  34         if (!s) {
  35                 throw runtime_error("read past the end of stream");
  36                 return;
  37         }
  38
  39         istream::char_type c;
  40         in.get(c);
  41         switch (c) {
  42                 case '{': case '}':
  43                 case '<': case '>':
  44                 case '[': case ']':
  45                 case '(': case ')':
  46                 case ';': case ':':
  47                 case ',': case '=':
  48                         current.type = Token::Type(c);
  49                         break;
  50                 case '+': case '-': case '.':
  51                 case '0': case '1': case '2': case '3': case '4':
  52                 case '5': case '6': case '7': case '8': case '9':
  53                         in.putback(c);
  54                         ReadNumber();
  55                         break;
  56                 case '"':
  57                         ReadString();
  58                         break;
  59                 case '#':
  60                 case '/':
  61                         in.putback(c);
  62                         ReadComment();
  63                         break;
  64                 default:
  65                         in.putback(c);
  66                         ReadIdentifier();
  67                         break;
  68         }
  69 }
  70
  71 namespace {
  72
  73 bool is_num_char(istream::char_type c) {
  74         return isxdigit(c)
  75                 || c == '.'
  76                 || c == '-'
  77                 || c == '+'
  78         ;
  79 }
  80
  81 }
  82
  83 void Tokenizer::ReadNumber() {
  84         current.type = Token::NUMBER;
  85         istream::char_type c;
  86         while (in.get(c)) {
  87                 if (is_num_char(c)) {
  88                         current.value += c;
  89                 } else {
  90                         in.putback(c);
  91                         break;
  92                 }
  93         }
  94 }
  95
  96 void Tokenizer::ReadString() {
  97         current.type = Token::STRING;
  98         bool escape = false;
  99
 100         istream::char_type c;
 101         while (in.get(c)) {
 102                 if (escape) {
 103                         escape = false;
 104                         switch (c) {
 105                                 case 'n':
 106                                         current.value += '\n';
 107                                         break;
 108                                 case 'r':
 109                                         current.value += '\t';
 110                                         break;
 111                                 case 't':
 112                                         current.value += '\t';
 113                                         break;
 114                                 default:
 115                                         current.value += c;
 116                                         break;
 117                         }
 118                 } else if (c == '"') {
 119                         break;
 120                 } else if (c == '\\') {
 121                         escape = true;
 122                 } else {
 123                         current.value += c;
 124                 }
 125         }
 126 }
 127
 128 void Tokenizer::ReadComment() {
 129         current.type = Token::COMMENT;
 130         istream::char_type c;
 131         in.get(c);
 132
 133         if (c == '#') {
 134                 while (in.get(c) && c != '\n') {
 135                         current.value += c;
 136                 }
 137                 return;
 138         }
 139
 140         // c is guaranteed to be '/' now
 141         if (!in.get(c)) {
 142                 throw runtime_error("unexpected end of stream");
 143         }
 144         if (c == '/') {
 145                 while (in.get(c) && c != '\n') {
 146                         current.value += c;
 147                 }
 148                 return;
 149         } else if (c != '*') {
 150                 throw runtime_error("invalid character after /");
 151         }
 152
 153         while (in.get(c)) {
 154                 if (c == '*') {
 155                         istream::char_type c2;
 156                         if (!in.get(c2)) {
 157                                 throw runtime_error("unexpected end of stream");
 158                         }
 159                         if (c2 == '/') {
 160                                 break;
 161                         } else {
 162                                 current.value += c;
 163                                 current.value += c2;
 164                         }
 165                 } else {
 166                         current.value += c;
 167                 }
 168         }
 169 }
 170
 171 void Tokenizer::ReadIdentifier() {
 172         current.type = Token::IDENTIFIER;
 173
 174         istream::char_type c;
 175         while (in.get(c)) {
 176                 if (isalnum(c) || c == '_') {
 177                         current.value += c;
 178                 } else {
 179                         in.putback(c);
 180                         break;
 181                 }
 182         }
 183 }
 184
 185
 186 TokenStreamReader::TokenStreamReader(istream &in)
 187 : in(in)
 188 , cached(false) {
 189
 190 }
 191
 192
 193 bool TokenStreamReader::HasMore() {
 194         if (cached) {
 195                 return true;
 196         }
 197         while (in.HasMore()) {
 198                 if (in.Next().type != Token::COMMENT) {
 199                         cached = true;
 200                         return true;
 201                 }
 202         }
 203         return false;
 204 }
 205
 206 const Token &TokenStreamReader::Next() {
 207         if (cached) {
 208                 cached = false;
 209                 return in.Current();
 210         } else {
 211                 return in.Next();
 212         }
 213 }
 214
 215 const Token &TokenStreamReader::Peek() {
 216         if (!cached) {
 217                 in.Next();
 218                 cached = true;
 219         }
 220         return in.Current();
 221 }
 222
 223
 224 void TokenStreamReader::Assert(Token::Type t) {
 225         if (GetType() != t) {
 226                 throw runtime_error("unexpected token in input stream");
 227         }
 228 }
 229
 230 Token::Type TokenStreamReader::GetType() const noexcept {
 231         return in.Current().type;
 232 }
 233
 234 const std::string &TokenStreamReader::GetValue() const noexcept {
 235         return in.Current().value;
 236 }
 237
 238 void TokenStreamReader::Skip(Token::Type t) {
 239         Next();
 240         Assert(t);
 241 }
 242
 243
 244 void TokenStreamReader::ReadBoolean(bool &b) {
 245         b = GetBool();
 246 }
 247
 248 void TokenStreamReader::ReadIdentifier(string &out) {
 249         Next();
 250         Assert(Token::IDENTIFIER);
 251         out = GetValue();
 252 }
 253
 254 void TokenStreamReader::ReadNumber(float &n) {
 255         n = GetFloat();
 256 }
 257
 258 void TokenStreamReader::ReadNumber(int &n) {
 259         n = GetInt();
 260 }
 261
 262 void TokenStreamReader::ReadNumber(unsigned long &n) {
 263         n = GetULong();
 264 }
 265
 266 void TokenStreamReader::ReadString(string &out) {
 267         Next();
 268         Assert(Token::STRING);
 269         out = GetValue();
 270 }
 271
 272
 273 void TokenStreamReader::ReadVec(glm::vec2 &v) {
 274         Skip(Token::BRACKET_OPEN);
 275         ReadNumber(v.x);
 276         Skip(Token::COMMA);
 277         ReadNumber(v.y);
 278         Skip(Token::BRACKET_CLOSE);
 279 }
 280
 281 void TokenStreamReader::ReadVec(glm::vec3 &v) {
 282         Skip(Token::BRACKET_OPEN);
 283         ReadNumber(v.x);
 284         Skip(Token::COMMA);
 285         ReadNumber(v.y);
 286         Skip(Token::COMMA);
 287         ReadNumber(v.z);
 288         Skip(Token::BRACKET_CLOSE);
 289 }
 290
 291 void TokenStreamReader::ReadVec(glm::vec4 &v) {
 292         Skip(Token::BRACKET_OPEN);
 293         ReadNumber(v.x);
 294         Skip(Token::COMMA);
 295         ReadNumber(v.y);
 296         Skip(Token::COMMA);
 297         ReadNumber(v.z);
 298         Skip(Token::COMMA);
 299         ReadNumber(v.w);
 300         Skip(Token::BRACKET_CLOSE);
 301 }
 302
 303 void TokenStreamReader::ReadVec(glm::ivec2 &v) {
 304         Skip(Token::BRACKET_OPEN);
 305         ReadNumber(v.x);
 306         Skip(Token::COMMA);
 307         ReadNumber(v.y);
 308         Skip(Token::BRACKET_CLOSE);
 309 }
 310
 311 void TokenStreamReader::ReadVec(glm::ivec3 &v) {
 312         Skip(Token::BRACKET_OPEN);
 313         ReadNumber(v.x);
 314         Skip(Token::COMMA);
 315         ReadNumber(v.y);
 316         Skip(Token::COMMA);
 317         ReadNumber(v.z);
 318         Skip(Token::BRACKET_CLOSE);
 319 }
 320
 321 void TokenStreamReader::ReadVec(glm::ivec4 &v) {
 322         Skip(Token::BRACKET_OPEN);
 323         ReadNumber(v.x);
 324         Skip(Token::COMMA);
 325         ReadNumber(v.y);
 326         Skip(Token::COMMA);
 327         ReadNumber(v.z);
 328         Skip(Token::COMMA);
 329         ReadNumber(v.w);
 330         Skip(Token::BRACKET_CLOSE);
 331 }
 332
 333
 334 bool TokenStreamReader::GetBool() {
 335         Next();
 336         switch (GetType()) {
 337                 case Token::NUMBER:
 338                         return GetInt() != 0;
 339                 case Token::IDENTIFIER:
 340                 case Token::STRING:
 341                         if (GetValue() == "true" || GetValue() == "yes" || GetValue() == "on") {
 342                                 return true;
 343                         } else if (GetValue() == "false" || GetValue() == "no" || GetValue() == "off") {
 344                                 return false;
 345                         } else {
 346                                 throw runtime_error("unexpected value in input stream");
 347                         }
 348                 default:
 349                         throw runtime_error("unexpected token in input stream");
 350         }
 351 }
 352
 353 float TokenStreamReader::GetFloat() {
 354         Next();
 355         Assert(Token::NUMBER);
 356         return stof(GetValue());
 357 }
 358
 359 int TokenStreamReader::GetInt() {
 360         Next();
 361         Assert(Token::NUMBER);
 362         return stoi(GetValue());
 363 }
 364
 365 unsigned long TokenStreamReader::GetULong() {
 366         Next();
 367         Assert(Token::NUMBER);
 368         return stoul(GetValue());
 369 }
 370
 371 }