aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/libraries/luajit-2.0/src/lj_lex.c
diff options
context:
space:
mode:
Diffstat (limited to 'libraries/luajit-2.0/src/lj_lex.c')
-rw-r--r--libraries/luajit-2.0/src/lj_lex.c508
1 files changed, 0 insertions, 508 deletions
diff --git a/libraries/luajit-2.0/src/lj_lex.c b/libraries/luajit-2.0/src/lj_lex.c
deleted file mode 100644
index 59d8225..0000000
--- a/libraries/luajit-2.0/src/lj_lex.c
+++ /dev/null
@@ -1,508 +0,0 @@
1/*
2** Lexical analyzer.
3** Copyright (C) 2005-2011 Mike Pall. See Copyright Notice in luajit.h
4**
5** Major portions taken verbatim or adapted from the Lua interpreter.
6** Copyright (C) 1994-2008 Lua.org, PUC-Rio. See Copyright Notice in lua.h
7*/
8
9#define lj_lex_c
10#define LUA_CORE
11
12#include "lj_obj.h"
13#include "lj_gc.h"
14#include "lj_err.h"
15#include "lj_str.h"
16#if LJ_HASFFI
17#include "lj_tab.h"
18#include "lj_ctype.h"
19#include "lj_cdata.h"
20#include "lualib.h"
21#endif
22#include "lj_state.h"
23#include "lj_lex.h"
24#include "lj_parse.h"
25#include "lj_char.h"
26
27/* Lua lexer token names. */
28static const char *const tokennames[] = {
29#define TKSTR1(name) #name,
30#define TKSTR2(name, sym) #sym,
31TKDEF(TKSTR1, TKSTR2)
32#undef TKSTR1
33#undef TKSTR2
34 NULL
35};
36
37/* -- Buffer handling ----------------------------------------------------- */
38
39#define char2int(c) ((int)(uint8_t)(c))
40#define next(ls) \
41 (ls->current = (ls->n--) > 0 ? char2int(*ls->p++) : fillbuf(ls))
42#define save_and_next(ls) (save(ls, ls->current), next(ls))
43#define currIsNewline(ls) (ls->current == '\n' || ls->current == '\r')
44#define END_OF_STREAM (-1)
45
46static int fillbuf(LexState *ls)
47{
48 size_t sz;
49 const char *buf = ls->rfunc(ls->L, ls->rdata, &sz);
50 if (buf == NULL || sz == 0) return END_OF_STREAM;
51 ls->n = (MSize)sz - 1;
52 ls->p = buf;
53 return char2int(*(ls->p++));
54}
55
56static LJ_NOINLINE void save_grow(LexState *ls, int c)
57{
58 MSize newsize;
59 if (ls->sb.sz >= LJ_MAX_STR/2)
60 lj_lex_error(ls, 0, LJ_ERR_XELEM);
61 newsize = ls->sb.sz * 2;
62 lj_str_resizebuf(ls->L, &ls->sb, newsize);
63 ls->sb.buf[ls->sb.n++] = (char)c;
64}
65
66static LJ_AINLINE void save(LexState *ls, int c)
67{
68 if (LJ_UNLIKELY(ls->sb.n + 1 > ls->sb.sz))
69 save_grow(ls, c);
70 else
71 ls->sb.buf[ls->sb.n++] = (char)c;
72}
73
74static void inclinenumber(LexState *ls)
75{
76 int old = ls->current;
77 lua_assert(currIsNewline(ls));
78 next(ls); /* skip `\n' or `\r' */
79 if (currIsNewline(ls) && ls->current != old)
80 next(ls); /* skip `\n\r' or `\r\n' */
81 if (++ls->linenumber >= LJ_MAX_LINE)
82 lj_lex_error(ls, ls->token, LJ_ERR_XLINES);
83}
84
85/* -- Scanner for terminals ----------------------------------------------- */
86
87#if LJ_HASFFI
88/* Load FFI library on-demand. Needed if we create cdata objects. */
89static void lex_loadffi(lua_State *L)
90{
91 ptrdiff_t oldtop = savestack(L, L->top);
92 luaopen_ffi(L);
93 L->top = restorestack(L, oldtop);
94}
95
96/* Parse 64 bit integer. */
97static int lex_number64(LexState *ls, TValue *tv)
98{
99 uint64_t n = 0;
100 uint8_t *p = (uint8_t *)ls->sb.buf;
101 CTypeID id = CTID_INT64;
102 GCcdata *cd;
103 int numl = 0;
104 if (p[0] == '0' && (p[1] & ~0x20) == 'X') { /* Hexadecimal. */
105 p += 2;
106 if (!lj_char_isxdigit(*p)) return 0;
107 do {
108 n = n*16 + (*p & 15);
109 if (!lj_char_isdigit(*p)) n += 9;
110 p++;
111 } while (lj_char_isxdigit(*p));
112 } else { /* Decimal. */
113 if (!lj_char_isdigit(*p)) return 0;
114 do {
115 n = n*10 + (*p - '0');
116 p++;
117 } while (lj_char_isdigit(*p));
118 }
119 for (;;) { /* Parse suffixes. */
120 if ((*p & ~0x20) == 'U')
121 id = CTID_UINT64;
122 else if ((*p & ~0x20) == 'L')
123 numl++;
124 else
125 break;
126 p++;
127 }
128 if (numl != 2 || *p != '\0') return 0;
129 /* Return cdata holding a 64 bit integer. */
130 cd = lj_cdata_new_(ls->L, id, 8);
131 *(uint64_t *)cdataptr(cd) = n;
132 lj_parse_keepcdata(ls, tv, cd);
133 return 1; /* Ok. */
134}
135#endif
136
137/* Parse a number literal. */
138static void lex_number(LexState *ls, TValue *tv)
139{
140 int c, xp = 'E';
141 lua_assert(lj_char_isdigit(ls->current));
142 if ((c = ls->current) == '0') {
143 save_and_next(ls);
144 if ((ls->current & ~0x20) == 'X') xp = 'P';
145 }
146 while (lj_char_isident(ls->current) || ls->current == '.' ||
147 ((ls->current == '-' || ls->current == '+') && (c & ~0x20) == xp)) {
148 c = ls->current;
149 save_and_next(ls);
150 }
151#if LJ_HASFFI
152 c &= ~0x20;
153 if ((c == 'I' || c == 'L' || c == 'U') && !ctype_ctsG(G(ls->L)))
154 lex_loadffi(ls->L);
155 if (c == 'I') /* Parse imaginary part of complex number. */
156 ls->sb.n--;
157#endif
158 save(ls, '\0');
159#if LJ_HASFFI
160 if ((c == 'L' || c == 'U') && lex_number64(ls, tv)) { /* Parse 64 bit int. */
161 return;
162 } else
163#endif
164 if (lj_str_numconv(ls->sb.buf, tv)) {
165#if LJ_HASFFI
166 if (c == 'I') { /* Return cdata holding a complex number. */
167 GCcdata *cd = lj_cdata_new_(ls->L, CTID_COMPLEX_DOUBLE, 2*sizeof(double));
168 ((double *)cdataptr(cd))[0] = 0;
169 ((double *)cdataptr(cd))[1] = numberVnum(tv);
170 lj_parse_keepcdata(ls, tv, cd);
171 }
172#endif
173 if (LJ_DUALNUM && tvisnum(tv)) {
174 int32_t k = lj_num2int(numV(tv));
175 if ((lua_Number)k == numV(tv)) /* -0 cannot end up here. */
176 setintV(tv, k);
177 }
178 return;
179 }
180 lj_lex_error(ls, TK_number, LJ_ERR_XNUMBER);
181}
182
183static int skip_sep(LexState *ls)
184{
185 int count = 0;
186 int s = ls->current;
187 lua_assert(s == '[' || s == ']');
188 save_and_next(ls);
189 while (ls->current == '=') {
190 save_and_next(ls);
191 count++;
192 }
193 return (ls->current == s) ? count : (-count) - 1;
194}
195
196static void read_long_string(LexState *ls, TValue *tv, int sep)
197{
198 save_and_next(ls); /* skip 2nd `[' */
199 if (currIsNewline(ls)) /* string starts with a newline? */
200 inclinenumber(ls); /* skip it */
201 for (;;) {
202 switch (ls->current) {
203 case END_OF_STREAM:
204 lj_lex_error(ls, TK_eof, tv ? LJ_ERR_XLSTR : LJ_ERR_XLCOM);
205 break;
206 case ']':
207 if (skip_sep(ls) == sep) {
208 save_and_next(ls); /* skip 2nd `]' */
209 goto endloop;
210 }
211 break;
212 case '\n':
213 case '\r':
214 save(ls, '\n');
215 inclinenumber(ls);
216 if (!tv) lj_str_resetbuf(&ls->sb); /* avoid wasting space */
217 break;
218 default:
219 if (tv) save_and_next(ls);
220 else next(ls);
221 break;
222 }
223 } endloop:
224 if (tv) {
225 GCstr *str = lj_parse_keepstr(ls, ls->sb.buf + (2 + (MSize)sep),
226 ls->sb.n - 2*(2 + (MSize)sep));
227 setstrV(ls->L, tv, str);
228 }
229}
230
231static void read_string(LexState *ls, int delim, TValue *tv)
232{
233 save_and_next(ls);
234 while (ls->current != delim) {
235 switch (ls->current) {
236 case END_OF_STREAM:
237 lj_lex_error(ls, TK_eof, LJ_ERR_XSTR);
238 continue;
239 case '\n':
240 case '\r':
241 lj_lex_error(ls, TK_string, LJ_ERR_XSTR);
242 continue;
243 case '\\': {
244 int c = next(ls); /* Skip the '\\'. */
245 switch (c) {
246 case 'a': c = '\a'; break;
247 case 'b': c = '\b'; break;
248 case 'f': c = '\f'; break;
249 case 'n': c = '\n'; break;
250 case 'r': c = '\r'; break;
251 case 't': c = '\t'; break;
252 case 'v': c = '\v'; break;
253 case 'x': /* Hexadecimal escape '\xXX'. */
254 c = (next(ls) & 15u) << 4;
255 if (!lj_char_isdigit(ls->current)) {
256 if (!lj_char_isxdigit(ls->current)) goto err_xesc;
257 c += 9 << 4;
258 }
259 c += (next(ls) & 15u);
260 if (!lj_char_isdigit(ls->current)) {
261 if (!lj_char_isxdigit(ls->current)) goto err_xesc;
262 c += 9;
263 }
264 break;
265 case 'z': /* Skip whitespace. */
266 next(ls);
267 while (lj_char_isspace(ls->current))
268 if (currIsNewline(ls)) inclinenumber(ls); else next(ls);
269 continue;
270 case '\n': case '\r': save(ls, '\n'); inclinenumber(ls); continue;
271 case '\\': case '\"': case '\'': break;
272 case END_OF_STREAM: continue;
273 default:
274 if (!lj_char_isdigit(c))
275 goto err_xesc;
276 c -= '0'; /* Decimal escape '\ddd'. */
277 if (lj_char_isdigit(next(ls))) {
278 c = c*10 + (ls->current - '0');
279 if (lj_char_isdigit(next(ls))) {
280 c = c*10 + (ls->current - '0');
281 if (c > 255) {
282 err_xesc:
283 lj_lex_error(ls, TK_string, LJ_ERR_XESC);
284 }
285 next(ls);
286 }
287 }
288 save(ls, c);
289 continue;
290 }
291 save(ls, c);
292 next(ls);
293 continue;
294 }
295 default:
296 save_and_next(ls);
297 break;
298 }
299 }
300 save_and_next(ls); /* skip delimiter */
301 setstrV(ls->L, tv, lj_parse_keepstr(ls, ls->sb.buf + 1, ls->sb.n - 2));
302}
303
304/* -- Main lexical scanner ------------------------------------------------ */
305
306static int llex(LexState *ls, TValue *tv)
307{
308 lj_str_resetbuf(&ls->sb);
309 for (;;) {
310 if (lj_char_isident(ls->current)) {
311 GCstr *s;
312 if (lj_char_isdigit(ls->current)) { /* Numeric literal. */
313 lex_number(ls, tv);
314 return TK_number;
315 }
316 /* Identifier or reserved word. */
317 do {
318 save_and_next(ls);
319 } while (lj_char_isident(ls->current));
320 s = lj_parse_keepstr(ls, ls->sb.buf, ls->sb.n);
321 if (s->reserved > 0) /* Reserved word? */
322 return TK_OFS + s->reserved;
323 setstrV(ls->L, tv, s);
324 return TK_name;
325 }
326 switch (ls->current) {
327 case '\n':
328 case '\r':
329 inclinenumber(ls);
330 continue;
331 case ' ':
332 case '\t':
333 case '\v':
334 case '\f':
335 next(ls);
336 continue;
337 case '-':
338 next(ls);
339 if (ls->current != '-') return '-';
340 /* else is a comment */
341 next(ls);
342 if (ls->current == '[') {
343 int sep = skip_sep(ls);
344 lj_str_resetbuf(&ls->sb); /* `skip_sep' may dirty the buffer */
345 if (sep >= 0) {
346 read_long_string(ls, NULL, sep); /* long comment */
347 lj_str_resetbuf(&ls->sb);
348 continue;
349 }
350 }
351 /* else short comment */
352 while (!currIsNewline(ls) && ls->current != END_OF_STREAM)
353 next(ls);
354 continue;
355 case '[': {
356 int sep = skip_sep(ls);
357 if (sep >= 0) {
358 read_long_string(ls, tv, sep);
359 return TK_string;
360 } else if (sep == -1) {
361 return '[';
362 } else {
363 lj_lex_error(ls, TK_string, LJ_ERR_XLDELIM);
364 continue;
365 }
366 }
367 case '=':
368 next(ls);
369 if (ls->current != '=') return '='; else { next(ls); return TK_eq; }
370 case '<':
371 next(ls);
372 if (ls->current != '=') return '<'; else { next(ls); return TK_le; }
373 case '>':
374 next(ls);
375 if (ls->current != '=') return '>'; else { next(ls); return TK_ge; }
376 case '~':
377 next(ls);
378 if (ls->current != '=') return '~'; else { next(ls); return TK_ne; }
379 case '"':
380 case '\'':
381 read_string(ls, ls->current, tv);
382 return TK_string;
383 case '.':
384 save_and_next(ls);
385 if (ls->current == '.') {
386 next(ls);
387 if (ls->current == '.') {
388 next(ls);
389 return TK_dots; /* ... */
390 }
391 return TK_concat; /* .. */
392 } else if (!lj_char_isdigit(ls->current)) {
393 return '.';
394 } else {
395 lex_number(ls, tv);
396 return TK_number;
397 }
398 case END_OF_STREAM:
399 return TK_eof;
400 default: {
401 int c = ls->current;
402 next(ls);
403 return c; /* Single-char tokens (+ - / ...). */
404 }
405 }
406 }
407}
408
409/* -- Lexer API ----------------------------------------------------------- */
410
411/* Setup lexer state. */
412int lj_lex_setup(lua_State *L, LexState *ls)
413{
414 ls->L = L;
415 ls->fs = NULL;
416 ls->n = 0;
417 ls->p = NULL;
418 ls->vstack = NULL;
419 ls->sizevstack = 0;
420 ls->vtop = 0;
421 ls->bcstack = NULL;
422 ls->sizebcstack = 0;
423 ls->lookahead = TK_eof; /* No look-ahead token. */
424 ls->linenumber = 1;
425 ls->lastline = 1;
426 lj_str_resizebuf(ls->L, &ls->sb, LJ_MIN_SBUF);
427 next(ls); /* Read-ahead first char. */
428 if (ls->current == 0xef && ls->n >= 2 && char2int(ls->p[0]) == 0xbb &&
429 char2int(ls->p[1]) == 0xbf) { /* Skip UTF-8 BOM (if buffered). */
430 ls->n -= 2;
431 ls->p += 2;
432 next(ls);
433 }
434 if (ls->current == '#') { /* Skip POSIX #! header line. */
435 do {
436 next(ls);
437 if (ls->current == END_OF_STREAM) return 0;
438 } while (!currIsNewline(ls));
439 inclinenumber(ls);
440 }
441 return (ls->current == LUA_SIGNATURE[0]); /* Bytecode dump? */
442}
443
444/* Cleanup lexer state. */
445void lj_lex_cleanup(lua_State *L, LexState *ls)
446{
447 global_State *g = G(L);
448 lj_mem_freevec(g, ls->bcstack, ls->sizebcstack, BCInsLine);
449 lj_mem_freevec(g, ls->vstack, ls->sizevstack, VarInfo);
450 lj_str_freebuf(g, &ls->sb);
451}
452
453void lj_lex_next(LexState *ls)
454{
455 ls->lastline = ls->linenumber;
456 if (LJ_LIKELY(ls->lookahead == TK_eof)) { /* No lookahead token? */
457 ls->token = llex(ls, &ls->tokenval); /* Get next token. */
458 } else { /* Otherwise return lookahead token. */
459 ls->token = ls->lookahead;
460 ls->lookahead = TK_eof;
461 ls->tokenval = ls->lookaheadval;
462 }
463}
464
465LexToken lj_lex_lookahead(LexState *ls)
466{
467 lua_assert(ls->lookahead == TK_eof);
468 ls->lookahead = llex(ls, &ls->lookaheadval);
469 return ls->lookahead;
470}
471
472const char *lj_lex_token2str(LexState *ls, LexToken token)
473{
474 if (token > TK_OFS)
475 return tokennames[token-TK_OFS-1];
476 else if (!lj_char_iscntrl(token))
477 return lj_str_pushf(ls->L, "%c", token);
478 else
479 return lj_str_pushf(ls->L, "char(%d)", token);
480}
481
482void lj_lex_error(LexState *ls, LexToken token, ErrMsg em, ...)
483{
484 const char *tok;
485 va_list argp;
486 if (token == 0) {
487 tok = NULL;
488 } else if (token == TK_name || token == TK_string || token == TK_number) {
489 save(ls, '\0');
490 tok = ls->sb.buf;
491 } else {
492 tok = lj_lex_token2str(ls, token);
493 }
494 va_start(argp, em);
495 lj_err_lex(ls->L, ls->chunkname, tok, ls->linenumber, em, argp);
496 va_end(argp);
497}
498
499void lj_lex_init(lua_State *L)
500{
501 uint32_t i;
502 for (i = 0; i < TK_RESERVED; i++) {
503 GCstr *s = lj_str_newz(L, tokennames[i]);
504 fixstring(s); /* Reserved words are never collected. */
505 s->reserved = (uint8_t)(i+1);
506 }
507}
508
generated by cgit v1.1 at 2024-07-12 04:46:23 +0000