src/io/token.cpp

   1 #include "Tokenizer.hpp"
   2 #include "TokenStreamReader.hpp"
   3
   4 #include <cctype>
   5 #include <istream>
   6 #include <stdexcept>
   7
   8 using namespace std;
   9
  10
  11 namespace blank {
  12
  13 Tokenizer::Tokenizer(istream &in)
  14 : in(in)
  15 , current() {
  16
  17 }
  18
  19
  20 bool Tokenizer::HasMore() {
  21         return bool(istream::sentry(in));
  22 }
  23
  24 const Token &Tokenizer::Next() {
  25         ReadToken();
  26         return Current();
  27 }
  28
  29 void Tokenizer::ReadToken() {
  30         current.type = Token::UNKNOWN;
  31         current.value.clear();
  32
  33         istream::sentry s(in);
  34         if (!s) {
  35                 // TODO: error?
  36                 return;
  37         }
  38
  39         istream::char_type c;
  40         in.get(c);
  41         switch (c) {
  42                 case '{': case '}':
  43                 case '<': case '>':
  44                 case '[': case ']':
  45                 case '(': case ')':
  46                 case ';': case ':':
  47                 case ',': case '=':
  48                         current.type = Token::Type(c);
  49                         break;
  50                 case '+': case '-':
  51                 case '0': case '1': case '2': case '3': case '4':
  52                 case '5': case '6': case '7': case '8': case '9':
  53                         in.putback(c);
  54                         ReadNumber();
  55                         break;
  56                 case '"':
  57                         ReadString();
  58                         break;
  59                 case '#':
  60                 case '/':
  61                         in.putback(c);
  62                         ReadComment();
  63                         break;
  64                 default:
  65                         in.putback(c);
  66                         ReadIdentifier();
  67                         break;
  68         }
  69 }
  70
  71 namespace {
  72
  73 bool is_num_char(istream::char_type c) {
  74         return isdigit(c)
  75                 || c == '.'
  76                 || c == '-'
  77                 || c == '+'
  78         ;
  79 }
  80
  81 }
  82
  83 void Tokenizer::ReadNumber() {
  84         current.type = Token::NUMBER;
  85         istream::char_type c;
  86         while (in.get(c)) {
  87                 if (is_num_char(c)) {
  88                         current.value += c;
  89                 } else {
  90                         in.putback(c);
  91                         break;
  92                 }
  93         }
  94 }
  95
  96 void Tokenizer::ReadString() {
  97         current.type = Token::STRING;
  98         bool escape = false;
  99
 100         istream::char_type c;
 101         while (in.get(c)) {
 102                 if (escape) {
 103                         escape = false;
 104                         switch (c) {
 105                                 case 'n':
 106                                         current.value += '\n';
 107                                         break;
 108                                 case 'r':
 109                                         current.value += '\t';
 110                                         break;
 111                                 case 't':
 112                                         current.value += '\t';
 113                                         break;
 114                                 default:
 115                                         current.value += c;
 116                                         break;
 117                         }
 118                 } else if (c == '"') {
 119                         break;
 120                 } else if (c == '\\') {
 121                         escape = true;
 122                 } else {
 123                         current.value += c;
 124                 }
 125         }
 126 }
 127
 128 void Tokenizer::ReadComment() {
 129         current.type = Token::COMMENT;
 130         istream::char_type c;
 131         in.get(c);
 132
 133         if (c == '#') {
 134                 while (in.get(c) && c != '\n') {
 135                         current.value += c;
 136                 }
 137                 return;
 138         }
 139
 140         // c is guaranteed to be '/' now
 141         if (!in.get(c)) {
 142                 throw runtime_error("unexpected end of stream");
 143         }
 144         if (c == '/') {
 145                 while (in.get(c) && c != '\n') {
 146                         current.value += c;
 147                 }
 148                 return;
 149         } else if (c != '*') {
 150                 throw runtime_error("invalid character after /");
 151         }
 152
 153         while (in.get(c)) {
 154                 if (c == '*') {
 155                         istream::char_type c2;
 156                         if (!in.get(c2)) {
 157                                 throw runtime_error("unexpected end of stream");
 158                         }
 159                         if (c2 == '/') {
 160                                 break;
 161                         } else {
 162                                 current.value += c;
 163                                 current.value += c2;
 164                         }
 165                 } else {
 166                         current.value += c;
 167                 }
 168         }
 169 }
 170
 171 void Tokenizer::ReadIdentifier() {
 172         current.type = Token::IDENTIFIER;
 173
 174         istream::char_type c;
 175         while (in.get(c)) {
 176                 if (isalnum(c) || c == '_') {
 177                         current.value += c;
 178                 } else {
 179                         in.putback(c);
 180                         break;
 181                 }
 182         }
 183 }
 184
 185
 186 TokenStreamReader::TokenStreamReader(istream &in)
 187 : in(in)
 188 , cached(false) {
 189
 190 }
 191
 192
 193 bool TokenStreamReader::HasMore() {
 194         while (in.HasMore()) {
 195                 if (in.Next().type != Token::COMMENT) {
 196                         cached = true;
 197                         return true;
 198                 }
 199         }
 200         return false;
 201 }
 202
 203 const Token &TokenStreamReader::Next() {
 204         if (cached) {
 205                 cached = false;
 206                 return in.Current();
 207         } else {
 208                 return in.Next();
 209         }
 210 }
 211
 212 const Token &TokenStreamReader::Peek() {
 213         if (!cached) {
 214                 in.Next();
 215                 cached = true;
 216         }
 217         return in.Current();
 218 }
 219
 220
 221 void TokenStreamReader::Assert(Token::Type t) {
 222         if (GetType() != t) {
 223                 throw runtime_error("unexpected token in input stream");
 224         }
 225 }
 226
 227 Token::Type TokenStreamReader::GetType() const noexcept {
 228         return in.Current().type;
 229 }
 230
 231 const std::string &TokenStreamReader::GetValue() const noexcept {
 232         return in.Current().value;
 233 }
 234
 235 void TokenStreamReader::Skip(Token::Type t) {
 236         Next();
 237         Assert(t);
 238 }
 239
 240
 241 void TokenStreamReader::ReadBoolean(bool &b) {
 242         b = GetBool();
 243 }
 244
 245 void TokenStreamReader::ReadIdentifier(string &out) {
 246         Next();
 247         Assert(Token::IDENTIFIER);
 248         out = GetValue();
 249 }
 250
 251 void TokenStreamReader::ReadNumber(float &n) {
 252         n = GetFloat();
 253 }
 254
 255 void TokenStreamReader::ReadNumber(int &n) {
 256         n = GetInt();
 257 }
 258
 259 void TokenStreamReader::ReadNumber(unsigned long &n) {
 260         n = GetULong();
 261 }
 262
 263 void TokenStreamReader::ReadString(string &out) {
 264         Next();
 265         Assert(Token::STRING);
 266         out = GetValue();
 267 }
 268
 269
 270 void TokenStreamReader::ReadVec(glm::vec2 &v) {
 271         Skip(Token::BRACKET_OPEN);
 272         ReadNumber(v.x);
 273         Skip(Token::COMMA);
 274         ReadNumber(v.y);
 275         Skip(Token::BRACKET_CLOSE);
 276 }
 277
 278 void TokenStreamReader::ReadVec(glm::vec3 &v) {
 279         Skip(Token::BRACKET_OPEN);
 280         ReadNumber(v.x);
 281         Skip(Token::COMMA);
 282         ReadNumber(v.y);
 283         Skip(Token::COMMA);
 284         ReadNumber(v.z);
 285         Skip(Token::BRACKET_CLOSE);
 286 }
 287
 288 void TokenStreamReader::ReadVec(glm::vec4 &v) {
 289         Skip(Token::BRACKET_OPEN);
 290         ReadNumber(v.x);
 291         Skip(Token::COMMA);
 292         ReadNumber(v.y);
 293         Skip(Token::COMMA);
 294         ReadNumber(v.z);
 295         Skip(Token::COMMA);
 296         ReadNumber(v.w);
 297         Skip(Token::BRACKET_CLOSE);
 298 }
 299
 300 void TokenStreamReader::ReadVec(glm::ivec2 &v) {
 301         Skip(Token::BRACKET_OPEN);
 302         ReadNumber(v.x);
 303         Skip(Token::COMMA);
 304         ReadNumber(v.y);
 305         Skip(Token::BRACKET_CLOSE);
 306 }
 307
 308 void TokenStreamReader::ReadVec(glm::ivec3 &v) {
 309         Skip(Token::BRACKET_OPEN);
 310         ReadNumber(v.x);
 311         Skip(Token::COMMA);
 312         ReadNumber(v.y);
 313         Skip(Token::COMMA);
 314         ReadNumber(v.z);
 315         Skip(Token::BRACKET_CLOSE);
 316 }
 317
 318 void TokenStreamReader::ReadVec(glm::ivec4 &v) {
 319         Skip(Token::BRACKET_OPEN);
 320         ReadNumber(v.x);
 321         Skip(Token::COMMA);
 322         ReadNumber(v.y);
 323         Skip(Token::COMMA);
 324         ReadNumber(v.z);
 325         Skip(Token::COMMA);
 326         ReadNumber(v.w);
 327         Skip(Token::BRACKET_CLOSE);
 328 }
 329
 330
 331 bool TokenStreamReader::GetBool() {
 332         Next();
 333         switch (GetType()) {
 334                 case Token::NUMBER:
 335                         return GetInt() != 0;
 336                 case Token::IDENTIFIER:
 337                 case Token::STRING:
 338                         if (GetValue() == "true" || GetValue() == "yes" || GetValue() == "on") {
 339                                 return true;
 340                         } else if (GetValue() == "false" || GetValue() == "no" || GetValue() == "off") {
 341                                 return false;
 342                         } else {
 343                                 throw runtime_error("unexpected value in input stream");
 344                         }
 345                 default:
 346                         throw runtime_error("unexpected token in input stream");
 347         }
 348 }
 349
 350 float TokenStreamReader::GetFloat() {
 351         Next();
 352         Assert(Token::NUMBER);
 353         return stof(GetValue());
 354 }
 355
 356 int TokenStreamReader::GetInt() {
 357         Next();
 358         Assert(Token::NUMBER);
 359         return stoi(GetValue());
 360 }
 361
 362 unsigned long TokenStreamReader::GetULong() {
 363         Next();
 364         Assert(Token::NUMBER);
 365         return stoul(GetValue());
 366 }
 367
 368 }