diff options
Diffstat (limited to '')
-rw-r--r-- | libraries/luajit-2.0/src/lj_lex.c | 505 |
1 files changed, 505 insertions, 0 deletions
diff --git a/libraries/luajit-2.0/src/lj_lex.c b/libraries/luajit-2.0/src/lj_lex.c new file mode 100644 index 0000000..00daccd --- /dev/null +++ b/libraries/luajit-2.0/src/lj_lex.c | |||
@@ -0,0 +1,505 @@ | |||
1 | /* | ||
2 | ** Lexical analyzer. | ||
3 | ** Copyright (C) 2005-2011 Mike Pall. See Copyright Notice in luajit.h | ||
4 | ** | ||
5 | ** Major portions taken verbatim or adapted from the Lua interpreter. | ||
6 | ** Copyright (C) 1994-2008 Lua.org, PUC-Rio. See Copyright Notice in lua.h | ||
7 | */ | ||
8 | |||
9 | #define lj_lex_c | ||
10 | #define LUA_CORE | ||
11 | |||
12 | #include "lj_obj.h" | ||
13 | #include "lj_gc.h" | ||
14 | #include "lj_err.h" | ||
15 | #include "lj_str.h" | ||
16 | #if LJ_HASFFI | ||
17 | #include "lj_tab.h" | ||
18 | #include "lj_ctype.h" | ||
19 | #include "lj_cdata.h" | ||
20 | #include "lualib.h" | ||
21 | #endif | ||
22 | #include "lj_state.h" | ||
23 | #include "lj_lex.h" | ||
24 | #include "lj_parse.h" | ||
25 | #include "lj_char.h" | ||
26 | |||
27 | /* Lua lexer token names. */ | ||
28 | static const char *const tokennames[] = { | ||
29 | #define TKSTR1(name) #name, | ||
30 | #define TKSTR2(name, sym) #sym, | ||
31 | TKDEF(TKSTR1, TKSTR2) | ||
32 | #undef TKSTR1 | ||
33 | #undef TKSTR2 | ||
34 | NULL | ||
35 | }; | ||
36 | |||
37 | /* -- Buffer handling ----------------------------------------------------- */ | ||
38 | |||
39 | #define char2int(c) ((int)(uint8_t)(c)) | ||
40 | #define next(ls) \ | ||
41 | (ls->current = (ls->n--) > 0 ? char2int(*ls->p++) : fillbuf(ls)) | ||
42 | #define save_and_next(ls) (save(ls, ls->current), next(ls)) | ||
43 | #define currIsNewline(ls) (ls->current == '\n' || ls->current == '\r') | ||
44 | #define END_OF_STREAM (-1) | ||
45 | |||
46 | static int fillbuf(LexState *ls) | ||
47 | { | ||
48 | size_t sz; | ||
49 | const char *buf = ls->rfunc(ls->L, ls->rdata, &sz); | ||
50 | if (buf == NULL || sz == 0) return END_OF_STREAM; | ||
51 | ls->n = (MSize)sz - 1; | ||
52 | ls->p = buf; | ||
53 | return char2int(*(ls->p++)); | ||
54 | } | ||
55 | |||
56 | static LJ_NOINLINE void save_grow(LexState *ls, int c) | ||
57 | { | ||
58 | MSize newsize; | ||
59 | if (ls->sb.sz >= LJ_MAX_STR/2) | ||
60 | lj_lex_error(ls, 0, LJ_ERR_XELEM); | ||
61 | newsize = ls->sb.sz * 2; | ||
62 | lj_str_resizebuf(ls->L, &ls->sb, newsize); | ||
63 | ls->sb.buf[ls->sb.n++] = (char)c; | ||
64 | } | ||
65 | |||
66 | static LJ_AINLINE void save(LexState *ls, int c) | ||
67 | { | ||
68 | if (LJ_UNLIKELY(ls->sb.n + 1 > ls->sb.sz)) | ||
69 | save_grow(ls, c); | ||
70 | else | ||
71 | ls->sb.buf[ls->sb.n++] = (char)c; | ||
72 | } | ||
73 | |||
74 | static void inclinenumber(LexState *ls) | ||
75 | { | ||
76 | int old = ls->current; | ||
77 | lua_assert(currIsNewline(ls)); | ||
78 | next(ls); /* skip `\n' or `\r' */ | ||
79 | if (currIsNewline(ls) && ls->current != old) | ||
80 | next(ls); /* skip `\n\r' or `\r\n' */ | ||
81 | if (++ls->linenumber >= LJ_MAX_LINE) | ||
82 | lj_lex_error(ls, ls->token, LJ_ERR_XLINES); | ||
83 | } | ||
84 | |||
85 | /* -- Scanner for terminals ----------------------------------------------- */ | ||
86 | |||
87 | #if LJ_HASFFI | ||
88 | /* Load FFI library on-demand. Needed if we create cdata objects. */ | ||
89 | static void lex_loadffi(lua_State *L) | ||
90 | { | ||
91 | ptrdiff_t oldtop = savestack(L, L->top); | ||
92 | luaopen_ffi(L); | ||
93 | L->top = restorestack(L, oldtop); | ||
94 | } | ||
95 | |||
96 | /* Parse 64 bit integer. */ | ||
97 | static int lex_number64(LexState *ls, TValue *tv) | ||
98 | { | ||
99 | uint64_t n = 0; | ||
100 | uint8_t *p = (uint8_t *)ls->sb.buf; | ||
101 | CTypeID id = CTID_INT64; | ||
102 | GCcdata *cd; | ||
103 | int numl = 0; | ||
104 | if (p[0] == '0' && (p[1] & ~0x20) == 'X') { /* Hexadecimal. */ | ||
105 | p += 2; | ||
106 | if (!lj_char_isxdigit(*p)) return 0; | ||
107 | do { | ||
108 | n = n*16 + (*p & 15); | ||
109 | if (!lj_char_isdigit(*p)) n += 9; | ||
110 | p++; | ||
111 | } while (lj_char_isxdigit(*p)); | ||
112 | } else { /* Decimal. */ | ||
113 | if (!lj_char_isdigit(*p)) return 0; | ||
114 | do { | ||
115 | n = n*10 + (*p - '0'); | ||
116 | p++; | ||
117 | } while (lj_char_isdigit(*p)); | ||
118 | } | ||
119 | for (;;) { /* Parse suffixes. */ | ||
120 | if ((*p & ~0x20) == 'U') | ||
121 | id = CTID_UINT64; | ||
122 | else if ((*p & ~0x20) == 'L') | ||
123 | numl++; | ||
124 | else | ||
125 | break; | ||
126 | p++; | ||
127 | } | ||
128 | if (numl != 2 || *p != '\0') return 0; | ||
129 | /* Return cdata holding a 64 bit integer. */ | ||
130 | cd = lj_cdata_new_(ls->L, id, 8); | ||
131 | *(uint64_t *)cdataptr(cd) = n; | ||
132 | lj_parse_keepcdata(ls, tv, cd); | ||
133 | return 1; /* Ok. */ | ||
134 | } | ||
135 | #endif | ||
136 | |||
137 | /* Parse a number literal. */ | ||
138 | static void lex_number(LexState *ls, TValue *tv) | ||
139 | { | ||
140 | int c; | ||
141 | lua_assert(lj_char_isdigit(ls->current)); | ||
142 | do { | ||
143 | c = ls->current; | ||
144 | save_and_next(ls); | ||
145 | } while (lj_char_isident(ls->current) || ls->current == '.' || | ||
146 | ((ls->current == '-' || ls->current == '+') && | ||
147 | ((c & ~0x20) == 'E' || (c & ~0x20) == 'P'))); | ||
148 | #if LJ_HASFFI | ||
149 | c &= ~0x20; | ||
150 | if ((c == 'I' || c == 'L' || c == 'U') && !ctype_ctsG(G(ls->L))) | ||
151 | lex_loadffi(ls->L); | ||
152 | if (c == 'I') /* Parse imaginary part of complex number. */ | ||
153 | ls->sb.n--; | ||
154 | #endif | ||
155 | save(ls, '\0'); | ||
156 | #if LJ_HASFFI | ||
157 | if ((c == 'L' || c == 'U') && lex_number64(ls, tv)) { /* Parse 64 bit int. */ | ||
158 | return; | ||
159 | } else | ||
160 | #endif | ||
161 | if (lj_str_numconv(ls->sb.buf, tv)) { | ||
162 | #if LJ_HASFFI | ||
163 | if (c == 'I') { /* Return cdata holding a complex number. */ | ||
164 | GCcdata *cd = lj_cdata_new_(ls->L, CTID_COMPLEX_DOUBLE, 2*sizeof(double)); | ||
165 | ((double *)cdataptr(cd))[0] = 0; | ||
166 | ((double *)cdataptr(cd))[1] = numberVnum(tv); | ||
167 | lj_parse_keepcdata(ls, tv, cd); | ||
168 | } | ||
169 | #endif | ||
170 | if (LJ_DUALNUM && tvisnum(tv)) { | ||
171 | int32_t k = lj_num2int(numV(tv)); | ||
172 | if ((lua_Number)k == numV(tv)) /* -0 cannot end up here. */ | ||
173 | setintV(tv, k); | ||
174 | } | ||
175 | return; | ||
176 | } | ||
177 | lj_lex_error(ls, TK_number, LJ_ERR_XNUMBER); | ||
178 | } | ||
179 | |||
180 | static int skip_sep(LexState *ls) | ||
181 | { | ||
182 | int count = 0; | ||
183 | int s = ls->current; | ||
184 | lua_assert(s == '[' || s == ']'); | ||
185 | save_and_next(ls); | ||
186 | while (ls->current == '=') { | ||
187 | save_and_next(ls); | ||
188 | count++; | ||
189 | } | ||
190 | return (ls->current == s) ? count : (-count) - 1; | ||
191 | } | ||
192 | |||
193 | static void read_long_string(LexState *ls, TValue *tv, int sep) | ||
194 | { | ||
195 | save_and_next(ls); /* skip 2nd `[' */ | ||
196 | if (currIsNewline(ls)) /* string starts with a newline? */ | ||
197 | inclinenumber(ls); /* skip it */ | ||
198 | for (;;) { | ||
199 | switch (ls->current) { | ||
200 | case END_OF_STREAM: | ||
201 | lj_lex_error(ls, TK_eof, tv ? LJ_ERR_XLSTR : LJ_ERR_XLCOM); | ||
202 | break; | ||
203 | case ']': | ||
204 | if (skip_sep(ls) == sep) { | ||
205 | save_and_next(ls); /* skip 2nd `]' */ | ||
206 | goto endloop; | ||
207 | } | ||
208 | break; | ||
209 | case '\n': | ||
210 | case '\r': | ||
211 | save(ls, '\n'); | ||
212 | inclinenumber(ls); | ||
213 | if (!tv) lj_str_resetbuf(&ls->sb); /* avoid wasting space */ | ||
214 | break; | ||
215 | default: | ||
216 | if (tv) save_and_next(ls); | ||
217 | else next(ls); | ||
218 | break; | ||
219 | } | ||
220 | } endloop: | ||
221 | if (tv) { | ||
222 | GCstr *str = lj_parse_keepstr(ls, ls->sb.buf + (2 + (MSize)sep), | ||
223 | ls->sb.n - 2*(2 + (MSize)sep)); | ||
224 | setstrV(ls->L, tv, str); | ||
225 | } | ||
226 | } | ||
227 | |||
228 | static void read_string(LexState *ls, int delim, TValue *tv) | ||
229 | { | ||
230 | save_and_next(ls); | ||
231 | while (ls->current != delim) { | ||
232 | switch (ls->current) { | ||
233 | case END_OF_STREAM: | ||
234 | lj_lex_error(ls, TK_eof, LJ_ERR_XSTR); | ||
235 | continue; | ||
236 | case '\n': | ||
237 | case '\r': | ||
238 | lj_lex_error(ls, TK_string, LJ_ERR_XSTR); | ||
239 | continue; | ||
240 | case '\\': { | ||
241 | int c = next(ls); /* Skip the '\\'. */ | ||
242 | switch (c) { | ||
243 | case 'a': c = '\a'; break; | ||
244 | case 'b': c = '\b'; break; | ||
245 | case 'f': c = '\f'; break; | ||
246 | case 'n': c = '\n'; break; | ||
247 | case 'r': c = '\r'; break; | ||
248 | case 't': c = '\t'; break; | ||
249 | case 'v': c = '\v'; break; | ||
250 | case 'x': /* Hexadecimal escape '\xXX'. */ | ||
251 | c = (next(ls) & 15u) << 4; | ||
252 | if (!lj_char_isdigit(ls->current)) { | ||
253 | if (!lj_char_isxdigit(ls->current)) goto err_xesc; | ||
254 | c += 9 << 4; | ||
255 | } | ||
256 | c += (next(ls) & 15u); | ||
257 | if (!lj_char_isdigit(ls->current)) { | ||
258 | if (!lj_char_isxdigit(ls->current)) goto err_xesc; | ||
259 | c += 9; | ||
260 | } | ||
261 | break; | ||
262 | case 'z': /* Skip whitespace. */ | ||
263 | next(ls); | ||
264 | while (lj_char_isspace(ls->current)) | ||
265 | if (currIsNewline(ls)) inclinenumber(ls); else next(ls); | ||
266 | continue; | ||
267 | case '\n': case '\r': save(ls, '\n'); inclinenumber(ls); continue; | ||
268 | case '\\': case '\"': case '\'': break; | ||
269 | case END_OF_STREAM: continue; | ||
270 | default: | ||
271 | if (!lj_char_isdigit(c)) | ||
272 | goto err_xesc; | ||
273 | c -= '0'; /* Decimal escape '\ddd'. */ | ||
274 | if (lj_char_isdigit(next(ls))) { | ||
275 | c = c*10 + (ls->current - '0'); | ||
276 | if (lj_char_isdigit(next(ls))) { | ||
277 | c = c*10 + (ls->current - '0'); | ||
278 | if (c > 255) { | ||
279 | err_xesc: | ||
280 | lj_lex_error(ls, TK_string, LJ_ERR_XESC); | ||
281 | } | ||
282 | next(ls); | ||
283 | } | ||
284 | } | ||
285 | save(ls, c); | ||
286 | continue; | ||
287 | } | ||
288 | save(ls, c); | ||
289 | next(ls); | ||
290 | continue; | ||
291 | } | ||
292 | default: | ||
293 | save_and_next(ls); | ||
294 | break; | ||
295 | } | ||
296 | } | ||
297 | save_and_next(ls); /* skip delimiter */ | ||
298 | setstrV(ls->L, tv, lj_parse_keepstr(ls, ls->sb.buf + 1, ls->sb.n - 2)); | ||
299 | } | ||
300 | |||
301 | /* -- Main lexical scanner ------------------------------------------------ */ | ||
302 | |||
303 | static int llex(LexState *ls, TValue *tv) | ||
304 | { | ||
305 | lj_str_resetbuf(&ls->sb); | ||
306 | for (;;) { | ||
307 | if (lj_char_isident(ls->current)) { | ||
308 | GCstr *s; | ||
309 | if (lj_char_isdigit(ls->current)) { /* Numeric literal. */ | ||
310 | lex_number(ls, tv); | ||
311 | return TK_number; | ||
312 | } | ||
313 | /* Identifier or reserved word. */ | ||
314 | do { | ||
315 | save_and_next(ls); | ||
316 | } while (lj_char_isident(ls->current)); | ||
317 | s = lj_parse_keepstr(ls, ls->sb.buf, ls->sb.n); | ||
318 | if (s->reserved > 0) /* Reserved word? */ | ||
319 | return TK_OFS + s->reserved; | ||
320 | setstrV(ls->L, tv, s); | ||
321 | return TK_name; | ||
322 | } | ||
323 | switch (ls->current) { | ||
324 | case '\n': | ||
325 | case '\r': | ||
326 | inclinenumber(ls); | ||
327 | continue; | ||
328 | case ' ': | ||
329 | case '\t': | ||
330 | case '\v': | ||
331 | case '\f': | ||
332 | next(ls); | ||
333 | continue; | ||
334 | case '-': | ||
335 | next(ls); | ||
336 | if (ls->current != '-') return '-'; | ||
337 | /* else is a comment */ | ||
338 | next(ls); | ||
339 | if (ls->current == '[') { | ||
340 | int sep = skip_sep(ls); | ||
341 | lj_str_resetbuf(&ls->sb); /* `skip_sep' may dirty the buffer */ | ||
342 | if (sep >= 0) { | ||
343 | read_long_string(ls, NULL, sep); /* long comment */ | ||
344 | lj_str_resetbuf(&ls->sb); | ||
345 | continue; | ||
346 | } | ||
347 | } | ||
348 | /* else short comment */ | ||
349 | while (!currIsNewline(ls) && ls->current != END_OF_STREAM) | ||
350 | next(ls); | ||
351 | continue; | ||
352 | case '[': { | ||
353 | int sep = skip_sep(ls); | ||
354 | if (sep >= 0) { | ||
355 | read_long_string(ls, tv, sep); | ||
356 | return TK_string; | ||
357 | } else if (sep == -1) { | ||
358 | return '['; | ||
359 | } else { | ||
360 | lj_lex_error(ls, TK_string, LJ_ERR_XLDELIM); | ||
361 | continue; | ||
362 | } | ||
363 | } | ||
364 | case '=': | ||
365 | next(ls); | ||
366 | if (ls->current != '=') return '='; else { next(ls); return TK_eq; } | ||
367 | case '<': | ||
368 | next(ls); | ||
369 | if (ls->current != '=') return '<'; else { next(ls); return TK_le; } | ||
370 | case '>': | ||
371 | next(ls); | ||
372 | if (ls->current != '=') return '>'; else { next(ls); return TK_ge; } | ||
373 | case '~': | ||
374 | next(ls); | ||
375 | if (ls->current != '=') return '~'; else { next(ls); return TK_ne; } | ||
376 | case '"': | ||
377 | case '\'': | ||
378 | read_string(ls, ls->current, tv); | ||
379 | return TK_string; | ||
380 | case '.': | ||
381 | save_and_next(ls); | ||
382 | if (ls->current == '.') { | ||
383 | next(ls); | ||
384 | if (ls->current == '.') { | ||
385 | next(ls); | ||
386 | return TK_dots; /* ... */ | ||
387 | } | ||
388 | return TK_concat; /* .. */ | ||
389 | } else if (!lj_char_isdigit(ls->current)) { | ||
390 | return '.'; | ||
391 | } else { | ||
392 | lex_number(ls, tv); | ||
393 | return TK_number; | ||
394 | } | ||
395 | case END_OF_STREAM: | ||
396 | return TK_eof; | ||
397 | default: { | ||
398 | int c = ls->current; | ||
399 | next(ls); | ||
400 | return c; /* Single-char tokens (+ - / ...). */ | ||
401 | } | ||
402 | } | ||
403 | } | ||
404 | } | ||
405 | |||
406 | /* -- Lexer API ----------------------------------------------------------- */ | ||
407 | |||
408 | /* Setup lexer state. */ | ||
409 | int lj_lex_setup(lua_State *L, LexState *ls) | ||
410 | { | ||
411 | ls->L = L; | ||
412 | ls->fs = NULL; | ||
413 | ls->n = 0; | ||
414 | ls->p = NULL; | ||
415 | ls->vstack = NULL; | ||
416 | ls->sizevstack = 0; | ||
417 | ls->vtop = 0; | ||
418 | ls->bcstack = NULL; | ||
419 | ls->sizebcstack = 0; | ||
420 | ls->lookahead = TK_eof; /* No look-ahead token. */ | ||
421 | ls->linenumber = 1; | ||
422 | ls->lastline = 1; | ||
423 | lj_str_resizebuf(ls->L, &ls->sb, LJ_MIN_SBUF); | ||
424 | next(ls); /* Read-ahead first char. */ | ||
425 | if (ls->current == 0xef && ls->n >= 2 && char2int(ls->p[0]) == 0xbb && | ||
426 | char2int(ls->p[1]) == 0xbf) { /* Skip UTF-8 BOM (if buffered). */ | ||
427 | ls->n -= 2; | ||
428 | ls->p += 2; | ||
429 | next(ls); | ||
430 | } | ||
431 | if (ls->current == '#') { /* Skip POSIX #! header line. */ | ||
432 | do { | ||
433 | next(ls); | ||
434 | if (ls->current == END_OF_STREAM) return 0; | ||
435 | } while (!currIsNewline(ls)); | ||
436 | inclinenumber(ls); | ||
437 | } | ||
438 | return (ls->current == LUA_SIGNATURE[0]); /* Bytecode dump? */ | ||
439 | } | ||
440 | |||
441 | /* Cleanup lexer state. */ | ||
442 | void lj_lex_cleanup(lua_State *L, LexState *ls) | ||
443 | { | ||
444 | global_State *g = G(L); | ||
445 | lj_mem_freevec(g, ls->bcstack, ls->sizebcstack, BCInsLine); | ||
446 | lj_mem_freevec(g, ls->vstack, ls->sizevstack, VarInfo); | ||
447 | lj_str_freebuf(g, &ls->sb); | ||
448 | } | ||
449 | |||
450 | void lj_lex_next(LexState *ls) | ||
451 | { | ||
452 | ls->lastline = ls->linenumber; | ||
453 | if (LJ_LIKELY(ls->lookahead == TK_eof)) { /* No lookahead token? */ | ||
454 | ls->token = llex(ls, &ls->tokenval); /* Get next token. */ | ||
455 | } else { /* Otherwise return lookahead token. */ | ||
456 | ls->token = ls->lookahead; | ||
457 | ls->lookahead = TK_eof; | ||
458 | ls->tokenval = ls->lookaheadval; | ||
459 | } | ||
460 | } | ||
461 | |||
462 | LexToken lj_lex_lookahead(LexState *ls) | ||
463 | { | ||
464 | lua_assert(ls->lookahead == TK_eof); | ||
465 | ls->lookahead = llex(ls, &ls->lookaheadval); | ||
466 | return ls->lookahead; | ||
467 | } | ||
468 | |||
469 | const char *lj_lex_token2str(LexState *ls, LexToken token) | ||
470 | { | ||
471 | if (token > TK_OFS) | ||
472 | return tokennames[token-TK_OFS-1]; | ||
473 | else if (!lj_char_iscntrl(token)) | ||
474 | return lj_str_pushf(ls->L, "%c", token); | ||
475 | else | ||
476 | return lj_str_pushf(ls->L, "char(%d)", token); | ||
477 | } | ||
478 | |||
479 | void lj_lex_error(LexState *ls, LexToken token, ErrMsg em, ...) | ||
480 | { | ||
481 | const char *tok; | ||
482 | va_list argp; | ||
483 | if (token == 0) { | ||
484 | tok = NULL; | ||
485 | } else if (token == TK_name || token == TK_string || token == TK_number) { | ||
486 | save(ls, '\0'); | ||
487 | tok = ls->sb.buf; | ||
488 | } else { | ||
489 | tok = lj_lex_token2str(ls, token); | ||
490 | } | ||
491 | va_start(argp, em); | ||
492 | lj_err_lex(ls->L, ls->chunkname, tok, ls->linenumber, em, argp); | ||
493 | va_end(argp); | ||
494 | } | ||
495 | |||
496 | void lj_lex_init(lua_State *L) | ||
497 | { | ||
498 | uint32_t i; | ||
499 | for (i = 0; i < TK_RESERVED; i++) { | ||
500 | GCstr *s = lj_str_newz(L, tokennames[i]); | ||
501 | fixstring(s); /* Reserved words are never collected. */ | ||
502 | s->reserved = (uint8_t)(i+1); | ||
503 | } | ||
504 | } | ||
505 | |||