]> git.localhorst.tv Git - l2e.git/blob - src/loader/Tokenizer.cpp
added Sort function to Inventory
[l2e.git] / src / loader / Tokenizer.cpp
1 /*
2  * Tokenizer.cpp
3  *
4  *  Created on: Aug 26, 2012
5  *      Author: holy
6  */
7
8 #include "Tokenizer.h"
9
10 #include <istream>
11
12 namespace loader {
13
14 bool Tokenizer::HasMore() {
15         if (isPutback) return true;
16         ScanSpace();
17         if (!in) return false;
18
19         putback = ReadToken();
20         isPutback = true;
21         if (!skipComments || putback.type != Token::COMMENT) return true;
22
23         while (in && putback.type == Token::COMMENT) {
24                 putback = ReadToken();
25                 ScanSpace();
26         }
27         return putback.type != Token::COMMENT;
28 }
29
30 void Tokenizer::ScanSpace() {
31         std::istream::char_type c;
32         in.get(c);
33         while (in && std::isspace(c)) {
34                 if (c == '\n') {
35                         ++line;
36                 }
37                 in.get(c);
38         }
39         if (in) {
40                 in.putback(c);
41         }
42 }
43
44 void Tokenizer::Putback(const Token &t) {
45         if (isPutback) {
46                 throw LexerError(line, "Tokenizer: double putback not supported");
47         } else {
48                 putback = t;
49                 isPutback = true;
50         }
51 }
52
53 const Tokenizer::Token &Tokenizer::Peek() {
54         if (!isPutback) {
55                 putback = GetNext();
56                 isPutback = true;
57         }
58         return putback;
59 }
60
61 Tokenizer::Token Tokenizer::GetNext() {
62         if (!HasMore()) {
63                 throw LexerError(line, "read beyond last token");
64         }
65         if (isPutback) {
66                 isPutback = false;
67                 return putback;
68         } else {
69                 return ReadToken();
70         }
71 }
72
73 Tokenizer::Token Tokenizer::ReadToken() {
74         ScanSpace();
75         std::istream::char_type c;
76         in.get(c);
77         switch (c) {
78                 case Token::ANGLE_BRACKET_OPEN:
79                 case Token::ANGLE_BRACKET_CLOSE:
80                 case Token::COLON:
81                 case Token::COMMA:
82                 case Token::BRACKET_OPEN:
83                 case Token::BRACKET_CLOSE:
84                 case Token::PARENTHESIS_OPEN:
85                 case Token::PARENTHESIS_CLOSE:
86                 case Token::COMMAND:
87                 case Token::REGISTER:
88                         return Token ((Token::Type) c);
89                 case '<': {
90                         std::istream::char_type c2;
91                         in.get(c2);
92                         if (c2 == '<') {
93                                 return Token(Token::SCRIPT_BEGIN);
94                         } else {
95                                 in.putback(c2);
96                                 return Token(Token::CHEVRON_OPEN);
97                         }
98                 }
99                 case '>': {
100                         std::istream::char_type c2;
101                         in.get(c2);
102                         if (c2 == '>') {
103                                 return Token(Token::SCRIPT_END);
104                         } else {
105                                 in.putback(c2);
106                                 return Token(Token::CHEVRON_CLOSE);
107                         }
108                 }
109                 case '+': case '-':
110                 case '0': case '1': case '2': case '3': case '4':
111                 case '5': case '6': case '7': case '8': case '9':
112                         in.putback(c);
113                         return ReadNumber();
114                 case '"':
115                         in.putback(c);
116                         return ReadString();
117                 case '/':
118                         {
119                                 std::istream::char_type c2;
120                                 in.get(c2);
121                                 if (c2 == '/') {
122                                         return ReadComment();
123                                 } else if (c2 == '*') {
124                                         return ReadMultilineComment();
125                                 } else {
126                                         throw LexerError(line, std::string("Tokenizer: cannot parse token: ") + c + c2 + ": expected / or *");
127                                 }
128                         }
129                         break;
130                 default:
131                         in.putback(c);
132                         {
133                                 Token t(ReadIdentifier());
134                                 if (std::isupper(c)) {
135                                         t.type = Token::TYPE_NAME;
136                                 } else if (std::islower(c)) {
137                                         CheckKeyword(t);
138                                 } else {
139                                         throw LexerError(line, std::string("Tokenizer: cannot parse token: ") + c);
140                                 }
141                                 return t;
142                         }
143         }
144 }
145
146 Tokenizer::Token Tokenizer::ReadNumber() {
147         Token t(Token::NUMBER);
148         bool isNegative(false);
149
150         std::istream::char_type c;
151         in.get(c);
152         if (c == '-') {
153                 isNegative = true;
154         } else if (c != '+') {
155                 in.putback(c);
156         }
157
158         while (in.get(c)) {
159                 if (!std::isdigit(c)) {
160                         in.putback(c);
161                         break;
162                 }
163                 t.number *= 10;
164                 t.number += c - '0';
165         }
166
167         if (isNegative) t.number *= -1;
168
169         return t;
170 }
171
172 Tokenizer::Token Tokenizer::ReadString() {
173         Token t(Token::STRING);
174         bool escape(false);
175
176         std::istream::char_type c;
177         in.get(c);
178         if (c != '"') {
179                 throw LexerError(line, "Tokenizer: strings must begin with '\"'");
180         }
181
182         while (in.get(c)) {
183                 if (escape) {
184                         escape = false;
185                         switch (c) {
186                                 case 'n':
187                                         t.str.push_back('\n');
188                                         break;
189                                 case 'r':
190                                         t.str.push_back('\r');
191                                         break;
192                                 case 't':
193                                         t.str.push_back('\t');
194                                         break;
195                                 default:
196                                         t.str.push_back(c);
197                                         break;
198                         }
199                 } else if (c == '"') {
200                         break;
201                 } else if (c == '\\') {
202                         escape = true;
203                 } else {
204                         t.str.push_back(c);
205                 }
206         }
207
208         return t;
209 }
210
211 Tokenizer::Token Tokenizer::ReadIdentifier() {
212         Token t(Token::IDENTIFIER);
213
214         std::istream::char_type c;
215         while (in.get(c)) {
216                 if (std::isalnum(c) || c == '_') {
217                         t.str.push_back(c);
218                 } else {
219                         in.putback(c);
220                         break;
221                 }
222         }
223
224         return t;
225 }
226
227 Tokenizer::Token Tokenizer::ReadComment() {
228         std::istream::char_type c;
229         while (in.get(c) && c != '\n');
230         ++line;
231         return Token(Token::COMMENT);
232 }
233
234 Tokenizer::Token Tokenizer::ReadMultilineComment() {
235         std::istream::char_type c;
236         while (in.get(c)) {
237                 if (c == '*') {
238                         std::istream::char_type c2;
239                         if (in.get(c2) && c2 == '/') {
240                                 break;
241                         }
242                 } else if (c == '\n') {
243                         ++line;
244                 }
245         }
246         return Token(Token::COMMENT);
247 }
248
249 bool Tokenizer::CheckKeyword(Token &t) {
250         if (t.str == "export") {
251                 t.type = Token::KEYWORD_EXPORT;
252                 return true;
253         } else if (t.str == "false") {
254                 t.type = Token::KEYWORD_FALSE;
255                 return true;
256         } else if (t.str == "include") {
257                 t.type = Token::KEYWORD_INCLUDE;
258                 return true;
259         } else if (t.str == "true") {
260                 t.type = Token::KEYWORD_TRUE;
261                 return true;
262         } else {
263                 return false;
264         }
265 }
266
267 }