]> git.localhorst.tv Git - blank.git/blob - src/io/token.cpp
give unique IDs to entities
[blank.git] / src / io / token.cpp
1 #include "Tokenizer.hpp"
2 #include "TokenStreamReader.hpp"
3
4 #include <cctype>
5 #include <istream>
6 #include <stdexcept>
7
8 using namespace std;
9
10
11 namespace blank {
12
13 Tokenizer::Tokenizer(istream &in)
14 : in(in)
15 , current() {
16
17 }
18
19
20 bool Tokenizer::HasMore() {
21         return bool(istream::sentry(in));
22 }
23
24 const Token &Tokenizer::Next() {
25         ReadToken();
26         return Current();
27 }
28
29 void Tokenizer::ReadToken() {
30         current.type = Token::UNKNOWN;
31         current.value.clear();
32
33         istream::sentry s(in);
34         if (!s) {
35                 throw runtime_error("read past the end of stream");
36                 return;
37         }
38
39         istream::char_type c;
40         in.get(c);
41         switch (c) {
42                 case '{': case '}':
43                 case '<': case '>':
44                 case '[': case ']':
45                 case '(': case ')':
46                 case ';': case ':':
47                 case ',': case '=':
48                         current.type = Token::Type(c);
49                         break;
50                 case '+': case '-': case '.':
51                 case '0': case '1': case '2': case '3': case '4':
52                 case '5': case '6': case '7': case '8': case '9':
53                         in.putback(c);
54                         ReadNumber();
55                         break;
56                 case '"':
57                         ReadString();
58                         break;
59                 case '#':
60                 case '/':
61                         in.putback(c);
62                         ReadComment();
63                         break;
64                 default:
65                         in.putback(c);
66                         ReadIdentifier();
67                         break;
68         }
69 }
70
71 namespace {
72
73 bool is_num_char(istream::char_type c) {
74         return isxdigit(c)
75                 || c == '.'
76                 || c == '-'
77                 || c == '+'
78         ;
79 }
80
81 }
82
83 void Tokenizer::ReadNumber() {
84         current.type = Token::NUMBER;
85         istream::char_type c;
86         while (in.get(c)) {
87                 if (is_num_char(c)) {
88                         current.value += c;
89                 } else {
90                         in.putback(c);
91                         break;
92                 }
93         }
94 }
95
96 void Tokenizer::ReadString() {
97         current.type = Token::STRING;
98         bool escape = false;
99
100         istream::char_type c;
101         while (in.get(c)) {
102                 if (escape) {
103                         escape = false;
104                         switch (c) {
105                                 case 'n':
106                                         current.value += '\n';
107                                         break;
108                                 case 'r':
109                                         current.value += '\t';
110                                         break;
111                                 case 't':
112                                         current.value += '\t';
113                                         break;
114                                 default:
115                                         current.value += c;
116                                         break;
117                         }
118                 } else if (c == '"') {
119                         break;
120                 } else if (c == '\\') {
121                         escape = true;
122                 } else {
123                         current.value += c;
124                 }
125         }
126 }
127
128 void Tokenizer::ReadComment() {
129         current.type = Token::COMMENT;
130         istream::char_type c;
131         in.get(c);
132
133         if (c == '#') {
134                 while (in.get(c) && c != '\n') {
135                         current.value += c;
136                 }
137                 return;
138         }
139
140         // c is guaranteed to be '/' now
141         if (!in.get(c)) {
142                 throw runtime_error("unexpected end of stream");
143         }
144         if (c == '/') {
145                 while (in.get(c) && c != '\n') {
146                         current.value += c;
147                 }
148                 return;
149         } else if (c != '*') {
150                 throw runtime_error("invalid character after /");
151         }
152
153         while (in.get(c)) {
154                 if (c == '*') {
155                         istream::char_type c2;
156                         if (!in.get(c2)) {
157                                 throw runtime_error("unexpected end of stream");
158                         }
159                         if (c2 == '/') {
160                                 break;
161                         } else {
162                                 current.value += c;
163                                 current.value += c2;
164                         }
165                 } else {
166                         current.value += c;
167                 }
168         }
169 }
170
171 void Tokenizer::ReadIdentifier() {
172         current.type = Token::IDENTIFIER;
173
174         istream::char_type c;
175         while (in.get(c)) {
176                 if (isalnum(c) || c == '_') {
177                         current.value += c;
178                 } else {
179                         in.putback(c);
180                         break;
181                 }
182         }
183 }
184
185
186 TokenStreamReader::TokenStreamReader(istream &in)
187 : in(in)
188 , cached(false) {
189
190 }
191
192
193 bool TokenStreamReader::HasMore() {
194         if (cached) {
195                 return true;
196         }
197         while (in.HasMore()) {
198                 if (in.Next().type != Token::COMMENT) {
199                         cached = true;
200                         return true;
201                 }
202         }
203         return false;
204 }
205
206 const Token &TokenStreamReader::Next() {
207         if (cached) {
208                 cached = false;
209                 return in.Current();
210         } else {
211                 return in.Next();
212         }
213 }
214
215 const Token &TokenStreamReader::Peek() {
216         if (!cached) {
217                 in.Next();
218                 cached = true;
219         }
220         return in.Current();
221 }
222
223
224 void TokenStreamReader::Assert(Token::Type t) {
225         if (GetType() != t) {
226                 throw runtime_error("unexpected token in input stream");
227         }
228 }
229
230 Token::Type TokenStreamReader::GetType() const noexcept {
231         return in.Current().type;
232 }
233
234 const std::string &TokenStreamReader::GetValue() const noexcept {
235         return in.Current().value;
236 }
237
238 void TokenStreamReader::Skip(Token::Type t) {
239         Next();
240         Assert(t);
241 }
242
243
244 void TokenStreamReader::ReadBoolean(bool &b) {
245         b = GetBool();
246 }
247
248 void TokenStreamReader::ReadIdentifier(string &out) {
249         Next();
250         Assert(Token::IDENTIFIER);
251         out = GetValue();
252 }
253
254 void TokenStreamReader::ReadNumber(float &n) {
255         n = GetFloat();
256 }
257
258 void TokenStreamReader::ReadNumber(int &n) {
259         n = GetInt();
260 }
261
262 void TokenStreamReader::ReadNumber(unsigned long &n) {
263         n = GetULong();
264 }
265
266 void TokenStreamReader::ReadString(string &out) {
267         Next();
268         Assert(Token::STRING);
269         out = GetValue();
270 }
271
272
273 void TokenStreamReader::ReadVec(glm::vec2 &v) {
274         Skip(Token::BRACKET_OPEN);
275         ReadNumber(v.x);
276         Skip(Token::COMMA);
277         ReadNumber(v.y);
278         Skip(Token::BRACKET_CLOSE);
279 }
280
281 void TokenStreamReader::ReadVec(glm::vec3 &v) {
282         Skip(Token::BRACKET_OPEN);
283         ReadNumber(v.x);
284         Skip(Token::COMMA);
285         ReadNumber(v.y);
286         Skip(Token::COMMA);
287         ReadNumber(v.z);
288         Skip(Token::BRACKET_CLOSE);
289 }
290
291 void TokenStreamReader::ReadVec(glm::vec4 &v) {
292         Skip(Token::BRACKET_OPEN);
293         ReadNumber(v.x);
294         Skip(Token::COMMA);
295         ReadNumber(v.y);
296         Skip(Token::COMMA);
297         ReadNumber(v.z);
298         Skip(Token::COMMA);
299         ReadNumber(v.w);
300         Skip(Token::BRACKET_CLOSE);
301 }
302
303 void TokenStreamReader::ReadVec(glm::ivec2 &v) {
304         Skip(Token::BRACKET_OPEN);
305         ReadNumber(v.x);
306         Skip(Token::COMMA);
307         ReadNumber(v.y);
308         Skip(Token::BRACKET_CLOSE);
309 }
310
311 void TokenStreamReader::ReadVec(glm::ivec3 &v) {
312         Skip(Token::BRACKET_OPEN);
313         ReadNumber(v.x);
314         Skip(Token::COMMA);
315         ReadNumber(v.y);
316         Skip(Token::COMMA);
317         ReadNumber(v.z);
318         Skip(Token::BRACKET_CLOSE);
319 }
320
321 void TokenStreamReader::ReadVec(glm::ivec4 &v) {
322         Skip(Token::BRACKET_OPEN);
323         ReadNumber(v.x);
324         Skip(Token::COMMA);
325         ReadNumber(v.y);
326         Skip(Token::COMMA);
327         ReadNumber(v.z);
328         Skip(Token::COMMA);
329         ReadNumber(v.w);
330         Skip(Token::BRACKET_CLOSE);
331 }
332
333
334 bool TokenStreamReader::GetBool() {
335         Next();
336         switch (GetType()) {
337                 case Token::NUMBER:
338                         return GetInt() != 0;
339                 case Token::IDENTIFIER:
340                 case Token::STRING:
341                         if (GetValue() == "true" || GetValue() == "yes" || GetValue() == "on") {
342                                 return true;
343                         } else if (GetValue() == "false" || GetValue() == "no" || GetValue() == "off") {
344                                 return false;
345                         } else {
346                                 throw runtime_error("unexpected value in input stream");
347                         }
348                 default:
349                         throw runtime_error("unexpected token in input stream");
350         }
351 }
352
353 float TokenStreamReader::GetFloat() {
354         Next();
355         Assert(Token::NUMBER);
356         return stof(GetValue());
357 }
358
359 int TokenStreamReader::GetInt() {
360         Next();
361         Assert(Token::NUMBER);
362         return stoi(GetValue());
363 }
364
365 unsigned long TokenStreamReader::GetULong() {
366         Next();
367         Assert(Token::NUMBER);
368         return stoul(GetValue());
369 }
370
371 }