]> git.localhorst.tv Git - blobs.git/blob - src/io/token.cpp
remote outdated TODO
[blobs.git] / src / io / token.cpp
1 #include "Token.hpp"
2 #include "Tokenizer.hpp"
3 #include "TokenStreamReader.hpp"
4
5 #include <cctype>
6 #include <istream>
7 #include <ostream>
8 #include <sstream>
9 #include <stdexcept>
10 #include <glm/gtc/quaternion.hpp>
11
12 using namespace std;
13
14
15 namespace blobs {
16 namespace io {
17
18 ostream &operator <<(ostream &out, Token::Type t) {
19         switch (t) {
20                 case Token::ANGLE_BRACKET_OPEN:
21                         return out << "ANGLE_BRACKET_OPEN";
22                 case Token::ANGLE_BRACKET_CLOSE:
23                         return out << "ANGLE_BRACKET_CLOSE";
24                 case Token::CHEVRON_OPEN:
25                         return out << "CHEVRON_OPEN";
26                 case Token::CHEVRON_CLOSE:
27                         return out << "CHEVRON_CLOSE";
28                 case Token::BRACKET_OPEN:
29                         return out << "BRACKET_OPEN";
30                 case Token::BRACKET_CLOSE:
31                         return out << "BRACKET_CLOSE";
32                 case Token::PARENTHESIS_OPEN:
33                         return out << "PARENTHESIS_OPEN";
34                 case Token::PARENTHESIS_CLOSE:
35                         return out << "PARENTHESIS_CLOSE";
36                 case Token::COLON:
37                         return out << "COLON";
38                 case Token::SEMICOLON:
39                         return out << "SEMICOLON";
40                 case Token::COMMA:
41                         return out << "COMMA";
42                 case Token::EQUALS:
43                         return out << "EQUALS";
44                 case Token::NUMBER:
45                         return out << "NUMBER";
46                 case Token::STRING:
47                         return out << "STRING";
48                 case Token::IDENTIFIER:
49                         return out << "IDENTIFIER";
50                 case Token::COMMENT:
51                         return out << "COMMENT";
52                 default:
53                         return out << "UNKNOWN";
54         }
55 }
56
57 ostream &operator <<(ostream &out, const Token &t) {
58         out << t.type;
59         switch (t.type) {
60                 case Token::UNKNOWN:
61                 case Token::NUMBER:
62                 case Token::STRING:
63                 case Token::IDENTIFIER:
64                 case Token::COMMENT:
65                         return out << '(' << t.value << ')';
66                 default:
67                         return out;
68         }
69 }
70
71 Tokenizer::Tokenizer(istream &in)
72 : in(in)
73 , current() {
74
75 }
76
77
78 bool Tokenizer::HasMore() {
79         return bool(istream::sentry(in));
80 }
81
82 const Token &Tokenizer::Next() {
83         ReadToken();
84         return Current();
85 }
86
87 void Tokenizer::ReadToken() {
88         current.type = Token::UNKNOWN;
89         current.value.clear();
90
91         istream::sentry s(in);
92         if (!s) {
93                 throw runtime_error("read past the end of stream");
94                 return;
95         }
96
97         istream::char_type c;
98         in.get(c);
99         switch (c) {
100                 case '{': case '}':
101                 case '<': case '>':
102                 case '[': case ']':
103                 case '(': case ')':
104                 case ';': case ':':
105                 case ',': case '=':
106                         current.type = Token::Type(c);
107                         break;
108                 case '+': case '-': case '.':
109                 case '0': case '1': case '2': case '3': case '4':
110                 case '5': case '6': case '7': case '8': case '9':
111                         in.putback(c);
112                         ReadNumber();
113                         break;
114                 case '"':
115                         ReadString();
116                         break;
117                 case '#':
118                 case '/':
119                         in.putback(c);
120                         ReadComment();
121                         break;
122                 default:
123                         in.putback(c);
124                         ReadIdentifier();
125                         break;
126         }
127 }
128
129 namespace {
130
131 bool is_num_char(istream::char_type c) {
132         return isxdigit(c)
133                 || c == '.'
134                 || c == '-'
135                 || c == '+'
136         ;
137 }
138
139 }
140
141 void Tokenizer::ReadNumber() {
142         current.type = Token::NUMBER;
143         istream::char_type c;
144         while (in.get(c)) {
145                 if (is_num_char(c)) {
146                         current.value += c;
147                 } else {
148                         in.putback(c);
149                         break;
150                 }
151         }
152 }
153
154 void Tokenizer::ReadString() {
155         current.type = Token::STRING;
156         bool escape = false;
157
158         istream::char_type c;
159         while (in.get(c)) {
160                 if (escape) {
161                         escape = false;
162                         switch (c) {
163                                 case 'n':
164                                         current.value += '\n';
165                                         break;
166                                 case 'r':
167                                         current.value += '\r';
168                                         break;
169                                 case 't':
170                                         current.value += '\t';
171                                         break;
172                                 default:
173                                         current.value += c;
174                                         break;
175                         }
176                 } else if (c == '"') {
177                         break;
178                 } else if (c == '\\') {
179                         escape = true;
180                 } else {
181                         current.value += c;
182                 }
183         }
184 }
185
186 void Tokenizer::ReadComment() {
187         current.type = Token::COMMENT;
188         istream::char_type c;
189         in.get(c);
190
191         if (c == '#') {
192                 while (in.get(c) && c != '\n') {
193                         current.value += c;
194                 }
195                 return;
196         }
197
198         // c is guaranteed to be '/' now
199         if (!in.get(c)) {
200                 throw runtime_error("unexpected end of stream");
201         }
202         if (c == '/') {
203                 while (in.get(c) && c != '\n') {
204                         current.value += c;
205                 }
206                 return;
207         } else if (c != '*') {
208                 throw runtime_error("invalid character after /");
209         }
210
211         while (in.get(c)) {
212                 if (c == '*') {
213                         istream::char_type c2;
214                         if (!in.get(c2)) {
215                                 throw runtime_error("unexpected end of stream");
216                         }
217                         if (c2 == '/') {
218                                 break;
219                         } else {
220                                 current.value += c;
221                                 current.value += c2;
222                         }
223                 } else {
224                         current.value += c;
225                 }
226         }
227 }
228
229 void Tokenizer::ReadIdentifier() {
230         current.type = Token::IDENTIFIER;
231
232         istream::char_type c;
233         while (in.get(c)) {
234                 if (isalnum(c) || c == '_' || c == '.') {
235                         current.value += c;
236                 } else {
237                         in.putback(c);
238                         break;
239                 }
240         }
241 }
242
243
244 TokenStreamReader::TokenStreamReader(istream &in)
245 : in(in)
246 , cached(false) {
247
248 }
249
250
251 bool TokenStreamReader::HasMore() {
252         if (cached) {
253                 return true;
254         }
255         SkipComments();
256         return cached;
257 }
258
259 const Token &TokenStreamReader::Next() {
260         SkipComments();
261         cached = false;
262         return in.Current();
263 }
264
265 void TokenStreamReader::SkipComments() {
266         if (cached) {
267                 if (in.Current().type == Token::COMMENT) {
268                         cached = false;
269                 } else {
270                         return;
271                 }
272         }
273         while (in.HasMore()) {
274                 if (in.Next().type != Token::COMMENT) {
275                         cached = true;
276                         return;
277                 }
278         }
279 }
280
281 const Token &TokenStreamReader::Peek() {
282         if (!cached) {
283                 Next();
284                 cached = true;
285         }
286         return in.Current();
287 }
288
289
290 void TokenStreamReader::Assert(Token::Type t) const {
291         if (GetType() != t) {
292                 stringstream s;
293                 s << "unexpected token in input stream: expected " << t << ", but got " << in.Current();
294                 throw runtime_error(s.str());
295         }
296 }
297
298 Token::Type TokenStreamReader::GetType() const noexcept {
299         return in.Current().type;
300 }
301
302 const std::string &TokenStreamReader::GetValue() const noexcept {
303         return in.Current().value;
304 }
305
306 void TokenStreamReader::Skip(Token::Type t) {
307         Next();
308         Assert(t);
309 }
310
311
312 void TokenStreamReader::ReadBoolean(bool &b) {
313         b = GetBool();
314 }
315
316 void TokenStreamReader::ReadIdentifier(string &out) {
317         Next();
318         Assert(Token::IDENTIFIER);
319         out = GetValue();
320 }
321
322 void TokenStreamReader::ReadNumber(double &n) {
323         n = GetDouble();
324 }
325
326 void TokenStreamReader::ReadNumber(float &n) {
327         n = GetFloat();
328 }
329
330 void TokenStreamReader::ReadNumber(int &n) {
331         n = GetInt();
332 }
333
334 void TokenStreamReader::ReadNumber(unsigned long &n) {
335         n = GetULong();
336 }
337
338 void TokenStreamReader::ReadString(string &out) {
339         Next();
340         Assert(Token::STRING);
341         out = GetValue();
342 }
343
344
345 void TokenStreamReader::ReadVec(glm::vec2 &v) {
346         Skip(Token::BRACKET_OPEN);
347         ReadNumber(v.x);
348         Skip(Token::COMMA);
349         ReadNumber(v.y);
350         Skip(Token::BRACKET_CLOSE);
351 }
352
353 void TokenStreamReader::ReadVec(glm::vec3 &v) {
354         Skip(Token::BRACKET_OPEN);
355         ReadNumber(v.x);
356         Skip(Token::COMMA);
357         ReadNumber(v.y);
358         Skip(Token::COMMA);
359         ReadNumber(v.z);
360         Skip(Token::BRACKET_CLOSE);
361 }
362
363 void TokenStreamReader::ReadVec(glm::vec4 &v) {
364         Skip(Token::BRACKET_OPEN);
365         ReadNumber(v.x);
366         Skip(Token::COMMA);
367         ReadNumber(v.y);
368         Skip(Token::COMMA);
369         ReadNumber(v.z);
370         Skip(Token::COMMA);
371         ReadNumber(v.w);
372         Skip(Token::BRACKET_CLOSE);
373 }
374
375 void TokenStreamReader::ReadVec(glm::dvec2 &v) {
376         Skip(Token::BRACKET_OPEN);
377         ReadNumber(v.x);
378         Skip(Token::COMMA);
379         ReadNumber(v.y);
380         Skip(Token::BRACKET_CLOSE);
381 }
382
383 void TokenStreamReader::ReadVec(glm::dvec3 &v) {
384         Skip(Token::BRACKET_OPEN);
385         ReadNumber(v.x);
386         Skip(Token::COMMA);
387         ReadNumber(v.y);
388         Skip(Token::COMMA);
389         ReadNumber(v.z);
390         Skip(Token::BRACKET_CLOSE);
391 }
392
393 void TokenStreamReader::ReadVec(glm::dvec4 &v) {
394         Skip(Token::BRACKET_OPEN);
395         ReadNumber(v.x);
396         Skip(Token::COMMA);
397         ReadNumber(v.y);
398         Skip(Token::COMMA);
399         ReadNumber(v.z);
400         Skip(Token::COMMA);
401         ReadNumber(v.w);
402         Skip(Token::BRACKET_CLOSE);
403 }
404
405 void TokenStreamReader::ReadVec(glm::ivec2 &v) {
406         Skip(Token::BRACKET_OPEN);
407         ReadNumber(v.x);
408         Skip(Token::COMMA);
409         ReadNumber(v.y);
410         Skip(Token::BRACKET_CLOSE);
411 }
412
413 void TokenStreamReader::ReadVec(glm::ivec3 &v) {
414         Skip(Token::BRACKET_OPEN);
415         ReadNumber(v.x);
416         Skip(Token::COMMA);
417         ReadNumber(v.y);
418         Skip(Token::COMMA);
419         ReadNumber(v.z);
420         Skip(Token::BRACKET_CLOSE);
421 }
422
423 void TokenStreamReader::ReadVec(glm::ivec4 &v) {
424         Skip(Token::BRACKET_OPEN);
425         ReadNumber(v.x);
426         Skip(Token::COMMA);
427         ReadNumber(v.y);
428         Skip(Token::COMMA);
429         ReadNumber(v.z);
430         Skip(Token::COMMA);
431         ReadNumber(v.w);
432         Skip(Token::BRACKET_CLOSE);
433 }
434
435 void TokenStreamReader::ReadQuat(glm::quat &q) {
436         Skip(Token::BRACKET_OPEN);
437         ReadNumber(q.w);
438         Skip(Token::COMMA);
439         ReadNumber(q.x);
440         Skip(Token::COMMA);
441         ReadNumber(q.y);
442         Skip(Token::COMMA);
443         ReadNumber(q.z);
444         Skip(Token::BRACKET_CLOSE);
445 }
446
447 void TokenStreamReader::ReadQuat(glm::dquat &q) {
448         Skip(Token::BRACKET_OPEN);
449         ReadNumber(q.w);
450         Skip(Token::COMMA);
451         ReadNumber(q.x);
452         Skip(Token::COMMA);
453         ReadNumber(q.y);
454         Skip(Token::COMMA);
455         ReadNumber(q.z);
456         Skip(Token::BRACKET_CLOSE);
457 }
458
459
460 bool TokenStreamReader::GetBool() {
461         Next();
462         return AsBool();
463 }
464
465 bool TokenStreamReader::AsBool() const {
466         switch (GetType()) {
467                 case Token::NUMBER:
468                         return AsInt() != 0;
469                 case Token::IDENTIFIER:
470                 case Token::STRING:
471                         if (GetValue() == "true" || GetValue() == "yes" || GetValue() == "on") {
472                                 return true;
473                         } else if (GetValue() == "false" || GetValue() == "no" || GetValue() == "off") {
474                                 return false;
475                         } else {
476                                 throw runtime_error("unexpected value in input stream: cannot cast " + GetValue() + " to bool");
477                         }
478                 default:
479                         {
480                                 stringstream s;
481                                 s << "unexpected token in input stream: cannot cast " << in.Current() << " to bool";
482                                 throw runtime_error(s.str());
483                         }
484         }
485 }
486
487 float TokenStreamReader::GetDouble() {
488         Next();
489         return AsDouble();
490 }
491
492 float TokenStreamReader::AsDouble() const {
493         Assert(Token::NUMBER);
494         return stod(GetValue());
495 }
496
497 float TokenStreamReader::GetFloat() {
498         Next();
499         return AsFloat();
500 }
501
502 float TokenStreamReader::AsFloat() const {
503         Assert(Token::NUMBER);
504         return stof(GetValue());
505 }
506
507 int TokenStreamReader::GetInt() {
508         Next();
509         return AsInt();
510 }
511
512 int TokenStreamReader::AsInt() const {
513         Assert(Token::NUMBER);
514         return stoi(GetValue());
515 }
516
517 unsigned long TokenStreamReader::GetULong() {
518         Next();
519         return AsULong();
520 }
521
522 unsigned long TokenStreamReader::AsULong() const {
523         Assert(Token::NUMBER);
524         return stoul(GetValue());
525 }
526
527 }
528 }