diff options
Diffstat (limited to '')
-rw-r--r-- | libraries/luajit-2.0/src/lj_trace.c | 812 |
1 files changed, 812 insertions, 0 deletions
diff --git a/libraries/luajit-2.0/src/lj_trace.c b/libraries/luajit-2.0/src/lj_trace.c new file mode 100644 index 0000000..9645ecb --- /dev/null +++ b/libraries/luajit-2.0/src/lj_trace.c | |||
@@ -0,0 +1,812 @@ | |||
1 | /* | ||
2 | ** Trace management. | ||
3 | ** Copyright (C) 2005-2011 Mike Pall. See Copyright Notice in luajit.h | ||
4 | */ | ||
5 | |||
6 | #define lj_trace_c | ||
7 | #define LUA_CORE | ||
8 | |||
9 | #include "lj_obj.h" | ||
10 | |||
11 | #if LJ_HASJIT | ||
12 | |||
13 | #include "lj_gc.h" | ||
14 | #include "lj_err.h" | ||
15 | #include "lj_debug.h" | ||
16 | #include "lj_str.h" | ||
17 | #include "lj_frame.h" | ||
18 | #include "lj_state.h" | ||
19 | #include "lj_bc.h" | ||
20 | #include "lj_ir.h" | ||
21 | #include "lj_jit.h" | ||
22 | #include "lj_iropt.h" | ||
23 | #include "lj_mcode.h" | ||
24 | #include "lj_trace.h" | ||
25 | #include "lj_snap.h" | ||
26 | #include "lj_gdbjit.h" | ||
27 | #include "lj_record.h" | ||
28 | #include "lj_asm.h" | ||
29 | #include "lj_dispatch.h" | ||
30 | #include "lj_vm.h" | ||
31 | #include "lj_vmevent.h" | ||
32 | #include "lj_target.h" | ||
33 | |||
34 | /* -- Error handling ------------------------------------------------------ */ | ||
35 | |||
36 | /* Synchronous abort with error message. */ | ||
37 | void lj_trace_err(jit_State *J, TraceError e) | ||
38 | { | ||
39 | setnilV(&J->errinfo); /* No error info. */ | ||
40 | setintV(J->L->top++, (int32_t)e); | ||
41 | lj_err_throw(J->L, LUA_ERRRUN); | ||
42 | } | ||
43 | |||
44 | /* Synchronous abort with error message and error info. */ | ||
45 | void lj_trace_err_info(jit_State *J, TraceError e) | ||
46 | { | ||
47 | setintV(J->L->top++, (int32_t)e); | ||
48 | lj_err_throw(J->L, LUA_ERRRUN); | ||
49 | } | ||
50 | |||
51 | /* -- Trace management ---------------------------------------------------- */ | ||
52 | |||
53 | /* The current trace is first assembled in J->cur. The variable length | ||
54 | ** arrays point to shared, growable buffers (J->irbuf etc.). When trace | ||
55 | ** recording ends successfully, the current trace and its data structures | ||
56 | ** are copied to a new (compact) GCtrace object. | ||
57 | */ | ||
58 | |||
59 | /* Find a free trace number. */ | ||
60 | static TraceNo trace_findfree(jit_State *J) | ||
61 | { | ||
62 | MSize osz, lim; | ||
63 | if (J->freetrace == 0) | ||
64 | J->freetrace = 1; | ||
65 | for (; J->freetrace < J->sizetrace; J->freetrace++) | ||
66 | if (traceref(J, J->freetrace) == NULL) | ||
67 | return J->freetrace++; | ||
68 | /* Need to grow trace array. */ | ||
69 | lim = (MSize)J->param[JIT_P_maxtrace] + 1; | ||
70 | if (lim < 2) lim = 2; else if (lim > 65535) lim = 65535; | ||
71 | osz = J->sizetrace; | ||
72 | if (osz >= lim) | ||
73 | return 0; /* Too many traces. */ | ||
74 | lj_mem_growvec(J->L, J->trace, J->sizetrace, lim, GCRef); | ||
75 | for (; osz < J->sizetrace; osz++) | ||
76 | setgcrefnull(J->trace[osz]); | ||
77 | return J->freetrace; | ||
78 | } | ||
79 | |||
80 | #define TRACE_APPENDVEC(field, szfield, tp) \ | ||
81 | T->field = (tp *)p; \ | ||
82 | memcpy(p, J->cur.field, J->cur.szfield*sizeof(tp)); \ | ||
83 | p += J->cur.szfield*sizeof(tp); | ||
84 | |||
85 | #ifdef LUAJIT_USE_PERFTOOLS | ||
86 | /* | ||
87 | ** Create symbol table of JIT-compiled code. For use with Linux perf tools. | ||
88 | ** Example usage: | ||
89 | ** perf record -f -e cycles luajit test.lua | ||
90 | ** perf report -s symbol | ||
91 | ** rm perf.data /tmp/perf-*.map | ||
92 | */ | ||
93 | #include <stdio.h> | ||
94 | #include <unistd.h> | ||
95 | |||
96 | static void perftools_addtrace(GCtrace *T) | ||
97 | { | ||
98 | static FILE *fp; | ||
99 | GCproto *pt = &gcref(T->startpt)->pt; | ||
100 | const BCIns *startpc = mref(T->startpc, const BCIns); | ||
101 | const char *name = proto_chunknamestr(pt); | ||
102 | BCLine lineno; | ||
103 | if (name[0] == '@' || name[0] == '=') | ||
104 | name++; | ||
105 | else | ||
106 | name = "(string)"; | ||
107 | lua_assert(startpc >= proto_bc(pt) && startpc < proto_bc(pt) + pt->sizebc); | ||
108 | lineno = lj_debug_line(pt, proto_bcpos(pt, startpc)); | ||
109 | if (!fp) { | ||
110 | char fname[40]; | ||
111 | sprintf(fname, "/tmp/perf-%d.map", getpid()); | ||
112 | if (!(fp = fopen(fname, "w"))) return; | ||
113 | setlinebuf(fp); | ||
114 | } | ||
115 | fprintf(fp, "%lx %x TRACE_%d::%s:%u\n", | ||
116 | (long)T->mcode, T->szmcode, T->traceno, name, lineno); | ||
117 | } | ||
118 | #endif | ||
119 | |||
120 | /* Save current trace by copying and compacting it. */ | ||
121 | static void trace_save(jit_State *J) | ||
122 | { | ||
123 | size_t sztr = ((sizeof(GCtrace)+7)&~7); | ||
124 | size_t szins = (J->cur.nins-J->cur.nk)*sizeof(IRIns); | ||
125 | size_t sz = sztr + szins + | ||
126 | J->cur.nsnap*sizeof(SnapShot) + | ||
127 | J->cur.nsnapmap*sizeof(SnapEntry); | ||
128 | GCtrace *T = lj_mem_newt(J->L, (MSize)sz, GCtrace); | ||
129 | char *p = (char *)T + sztr; | ||
130 | memcpy(T, &J->cur, sizeof(GCtrace)); | ||
131 | setgcrefr(T->nextgc, J2G(J)->gc.root); | ||
132 | setgcrefp(J2G(J)->gc.root, T); | ||
133 | newwhite(J2G(J), T); | ||
134 | T->gct = ~LJ_TTRACE; | ||
135 | T->ir = (IRIns *)p - J->cur.nk; | ||
136 | memcpy(p, J->cur.ir+J->cur.nk, szins); | ||
137 | p += szins; | ||
138 | TRACE_APPENDVEC(snap, nsnap, SnapShot) | ||
139 | TRACE_APPENDVEC(snapmap, nsnapmap, SnapEntry) | ||
140 | J->cur.traceno = 0; | ||
141 | setgcrefp(J->trace[T->traceno], T); | ||
142 | lj_gc_barriertrace(J2G(J), T->traceno); | ||
143 | lj_gdbjit_addtrace(J, T); | ||
144 | #ifdef LUAJIT_USE_PERFTOOLS | ||
145 | perftools_addtrace(T); | ||
146 | #endif | ||
147 | } | ||
148 | |||
149 | void LJ_FASTCALL lj_trace_free(global_State *g, GCtrace *T) | ||
150 | { | ||
151 | jit_State *J = G2J(g); | ||
152 | if (T->traceno) { | ||
153 | lj_gdbjit_deltrace(J, T); | ||
154 | if (T->traceno < J->freetrace) | ||
155 | J->freetrace = T->traceno; | ||
156 | setgcrefnull(J->trace[T->traceno]); | ||
157 | } | ||
158 | lj_mem_free(g, T, | ||
159 | ((sizeof(GCtrace)+7)&~7) + (T->nins-T->nk)*sizeof(IRIns) + | ||
160 | T->nsnap*sizeof(SnapShot) + T->nsnapmap*sizeof(SnapEntry)); | ||
161 | } | ||
162 | |||
163 | /* Re-enable compiling a prototype by unpatching any modified bytecode. */ | ||
164 | void lj_trace_reenableproto(GCproto *pt) | ||
165 | { | ||
166 | if ((pt->flags & PROTO_ILOOP)) { | ||
167 | BCIns *bc = proto_bc(pt); | ||
168 | BCPos i, sizebc = pt->sizebc;; | ||
169 | pt->flags &= ~PROTO_ILOOP; | ||
170 | if (bc_op(bc[0]) == BC_IFUNCF) | ||
171 | setbc_op(&bc[0], BC_FUNCF); | ||
172 | for (i = 1; i < sizebc; i++) { | ||
173 | BCOp op = bc_op(bc[i]); | ||
174 | if (op == BC_IFORL || op == BC_IITERL || op == BC_ILOOP) | ||
175 | setbc_op(&bc[i], (int)op+(int)BC_LOOP-(int)BC_ILOOP); | ||
176 | } | ||
177 | } | ||
178 | } | ||
179 | |||
180 | /* Unpatch the bytecode modified by a root trace. */ | ||
181 | static void trace_unpatch(jit_State *J, GCtrace *T) | ||
182 | { | ||
183 | BCOp op = bc_op(T->startins); | ||
184 | BCIns *pc = mref(T->startpc, BCIns); | ||
185 | UNUSED(J); | ||
186 | if (op == BC_JMP) | ||
187 | return; /* No need to unpatch branches in parent traces (yet). */ | ||
188 | switch (bc_op(*pc)) { | ||
189 | case BC_JFORL: | ||
190 | lua_assert(traceref(J, bc_d(*pc)) == T); | ||
191 | *pc = T->startins; | ||
192 | pc += bc_j(T->startins); | ||
193 | lua_assert(bc_op(*pc) == BC_JFORI); | ||
194 | setbc_op(pc, BC_FORI); | ||
195 | break; | ||
196 | case BC_JITERL: | ||
197 | case BC_JLOOP: | ||
198 | lua_assert(op == BC_ITERL || op == BC_LOOP || bc_isret(op)); | ||
199 | *pc = T->startins; | ||
200 | break; | ||
201 | case BC_JMP: | ||
202 | lua_assert(op == BC_ITERL); | ||
203 | pc += bc_j(*pc)+2; | ||
204 | if (bc_op(*pc) == BC_JITERL) { | ||
205 | lua_assert(traceref(J, bc_d(*pc)) == T); | ||
206 | *pc = T->startins; | ||
207 | } | ||
208 | break; | ||
209 | case BC_JFUNCF: | ||
210 | lua_assert(op == BC_FUNCF); | ||
211 | *pc = T->startins; | ||
212 | break; | ||
213 | default: /* Already unpatched. */ | ||
214 | break; | ||
215 | } | ||
216 | } | ||
217 | |||
218 | /* Flush a root trace. */ | ||
219 | static void trace_flushroot(jit_State *J, GCtrace *T) | ||
220 | { | ||
221 | GCproto *pt = &gcref(T->startpt)->pt; | ||
222 | lua_assert(T->root == 0 && pt != NULL); | ||
223 | /* First unpatch any modified bytecode. */ | ||
224 | trace_unpatch(J, T); | ||
225 | /* Unlink root trace from chain anchored in prototype. */ | ||
226 | if (pt->trace == T->traceno) { /* Trace is first in chain. Easy. */ | ||
227 | pt->trace = T->nextroot; | ||
228 | } else if (pt->trace) { /* Otherwise search in chain of root traces. */ | ||
229 | GCtrace *T2 = traceref(J, pt->trace); | ||
230 | if (T2) { | ||
231 | for (; T2->nextroot; T2 = traceref(J, T2->nextroot)) | ||
232 | if (T2->nextroot == T->traceno) { | ||
233 | T2->nextroot = T->nextroot; /* Unlink from chain. */ | ||
234 | break; | ||
235 | } | ||
236 | } | ||
237 | } | ||
238 | } | ||
239 | |||
240 | /* Flush a trace. Only root traces are considered. */ | ||
241 | void lj_trace_flush(jit_State *J, TraceNo traceno) | ||
242 | { | ||
243 | if (traceno > 0 && traceno < J->sizetrace) { | ||
244 | GCtrace *T = traceref(J, traceno); | ||
245 | if (T && T->root == 0) | ||
246 | trace_flushroot(J, T); | ||
247 | } | ||
248 | } | ||
249 | |||
250 | /* Flush all traces associated with a prototype. */ | ||
251 | void lj_trace_flushproto(global_State *g, GCproto *pt) | ||
252 | { | ||
253 | while (pt->trace != 0) | ||
254 | trace_flushroot(G2J(g), traceref(G2J(g), pt->trace)); | ||
255 | } | ||
256 | |||
257 | /* Flush all traces. */ | ||
258 | int lj_trace_flushall(lua_State *L) | ||
259 | { | ||
260 | jit_State *J = L2J(L); | ||
261 | ptrdiff_t i; | ||
262 | if ((J2G(J)->hookmask & HOOK_GC)) | ||
263 | return 1; | ||
264 | for (i = (ptrdiff_t)J->sizetrace-1; i > 0; i--) { | ||
265 | GCtrace *T = traceref(J, i); | ||
266 | if (T) { | ||
267 | if (T->root == 0) | ||
268 | trace_flushroot(J, T); | ||
269 | lj_gdbjit_deltrace(J, T); | ||
270 | T->traceno = 0; | ||
271 | setgcrefnull(J->trace[i]); | ||
272 | } | ||
273 | } | ||
274 | J->cur.traceno = 0; | ||
275 | J->freetrace = 0; | ||
276 | /* Clear penalty cache. */ | ||
277 | memset(J->penalty, 0, sizeof(J->penalty)); | ||
278 | /* Free the whole machine code and invalidate all exit stub groups. */ | ||
279 | lj_mcode_free(J); | ||
280 | memset(J->exitstubgroup, 0, sizeof(J->exitstubgroup)); | ||
281 | lj_vmevent_send(L, TRACE, | ||
282 | setstrV(L, L->top++, lj_str_newlit(L, "flush")); | ||
283 | ); | ||
284 | return 0; | ||
285 | } | ||
286 | |||
287 | /* Initialize JIT compiler state. */ | ||
288 | void lj_trace_initstate(global_State *g) | ||
289 | { | ||
290 | jit_State *J = G2J(g); | ||
291 | TValue *tv; | ||
292 | /* Initialize SIMD constants. */ | ||
293 | tv = LJ_KSIMD(J, LJ_KSIMD_ABS); | ||
294 | tv[0].u64 = U64x(7fffffff,ffffffff); | ||
295 | tv[1].u64 = U64x(7fffffff,ffffffff); | ||
296 | tv = LJ_KSIMD(J, LJ_KSIMD_NEG); | ||
297 | tv[0].u64 = U64x(80000000,00000000); | ||
298 | tv[1].u64 = U64x(80000000,00000000); | ||
299 | } | ||
300 | |||
301 | /* Free everything associated with the JIT compiler state. */ | ||
302 | void lj_trace_freestate(global_State *g) | ||
303 | { | ||
304 | jit_State *J = G2J(g); | ||
305 | #ifdef LUA_USE_ASSERT | ||
306 | { /* This assumes all traces have already been freed. */ | ||
307 | ptrdiff_t i; | ||
308 | for (i = 1; i < (ptrdiff_t)J->sizetrace; i++) | ||
309 | lua_assert(i == (ptrdiff_t)J->cur.traceno || traceref(J, i) == NULL); | ||
310 | } | ||
311 | #endif | ||
312 | lj_mcode_free(J); | ||
313 | lj_ir_k64_freeall(J); | ||
314 | lj_mem_freevec(g, J->snapmapbuf, J->sizesnapmap, SnapEntry); | ||
315 | lj_mem_freevec(g, J->snapbuf, J->sizesnap, SnapShot); | ||
316 | lj_mem_freevec(g, J->irbuf + J->irbotlim, J->irtoplim - J->irbotlim, IRIns); | ||
317 | lj_mem_freevec(g, J->trace, J->sizetrace, GCRef); | ||
318 | } | ||
319 | |||
320 | /* -- Penalties and blacklisting ------------------------------------------ */ | ||
321 | |||
322 | /* Blacklist a bytecode instruction. */ | ||
323 | static void blacklist_pc(GCproto *pt, BCIns *pc) | ||
324 | { | ||
325 | setbc_op(pc, (int)bc_op(*pc)+(int)BC_ILOOP-(int)BC_LOOP); | ||
326 | pt->flags |= PROTO_ILOOP; | ||
327 | } | ||
328 | |||
329 | /* Penalize a bytecode instruction. */ | ||
330 | static void penalty_pc(jit_State *J, GCproto *pt, BCIns *pc, TraceError e) | ||
331 | { | ||
332 | uint32_t i, val = PENALTY_MIN; | ||
333 | for (i = 0; i < PENALTY_SLOTS; i++) | ||
334 | if (mref(J->penalty[i].pc, const BCIns) == pc) { /* Cache slot found? */ | ||
335 | /* First try to bump its hotcount several times. */ | ||
336 | val = ((uint32_t)J->penalty[i].val << 1) + | ||
337 | LJ_PRNG_BITS(J, PENALTY_RNDBITS); | ||
338 | if (val > PENALTY_MAX) { | ||
339 | blacklist_pc(pt, pc); /* Blacklist it, if that didn't help. */ | ||
340 | return; | ||
341 | } | ||
342 | goto setpenalty; | ||
343 | } | ||
344 | /* Assign a new penalty cache slot. */ | ||
345 | i = J->penaltyslot; | ||
346 | J->penaltyslot = (J->penaltyslot + 1) & (PENALTY_SLOTS-1); | ||
347 | setmref(J->penalty[i].pc, pc); | ||
348 | setpenalty: | ||
349 | J->penalty[i].val = (uint16_t)val; | ||
350 | J->penalty[i].reason = e; | ||
351 | hotcount_set(J2GG(J), pc+1, val); | ||
352 | } | ||
353 | |||
354 | /* -- Trace compiler state machine ---------------------------------------- */ | ||
355 | |||
356 | /* Start tracing. */ | ||
357 | static void trace_start(jit_State *J) | ||
358 | { | ||
359 | lua_State *L; | ||
360 | TraceNo traceno; | ||
361 | |||
362 | if ((J->pt->flags & PROTO_NOJIT)) { /* JIT disabled for this proto? */ | ||
363 | if (J->parent == 0) { | ||
364 | /* Lazy bytecode patching to disable hotcount events. */ | ||
365 | lua_assert(bc_op(*J->pc) == BC_FORL || bc_op(*J->pc) == BC_ITERL || | ||
366 | bc_op(*J->pc) == BC_LOOP || bc_op(*J->pc) == BC_FUNCF); | ||
367 | setbc_op(J->pc, (int)bc_op(*J->pc)+(int)BC_ILOOP-(int)BC_LOOP); | ||
368 | J->pt->flags |= PROTO_ILOOP; | ||
369 | } | ||
370 | J->state = LJ_TRACE_IDLE; /* Silently ignored. */ | ||
371 | return; | ||
372 | } | ||
373 | |||
374 | /* Get a new trace number. */ | ||
375 | traceno = trace_findfree(J); | ||
376 | if (LJ_UNLIKELY(traceno == 0)) { /* No free trace? */ | ||
377 | lua_assert((J2G(J)->hookmask & HOOK_GC) == 0); | ||
378 | lj_trace_flushall(J->L); | ||
379 | J->state = LJ_TRACE_IDLE; /* Silently ignored. */ | ||
380 | return; | ||
381 | } | ||
382 | setgcrefp(J->trace[traceno], &J->cur); | ||
383 | |||
384 | /* Setup enough of the current trace to be able to send the vmevent. */ | ||
385 | memset(&J->cur, 0, sizeof(GCtrace)); | ||
386 | J->cur.traceno = traceno; | ||
387 | J->cur.nins = J->cur.nk = REF_BASE; | ||
388 | J->cur.ir = J->irbuf; | ||
389 | J->cur.snap = J->snapbuf; | ||
390 | J->cur.snapmap = J->snapmapbuf; | ||
391 | J->mergesnap = 0; | ||
392 | J->needsnap = 0; | ||
393 | J->bcskip = 0; | ||
394 | J->guardemit.irt = 0; | ||
395 | J->postproc = LJ_POST_NONE; | ||
396 | lj_resetsplit(J); | ||
397 | setgcref(J->cur.startpt, obj2gco(J->pt)); | ||
398 | |||
399 | L = J->L; | ||
400 | lj_vmevent_send(L, TRACE, | ||
401 | setstrV(L, L->top++, lj_str_newlit(L, "start")); | ||
402 | setintV(L->top++, traceno); | ||
403 | setfuncV(L, L->top++, J->fn); | ||
404 | setintV(L->top++, proto_bcpos(J->pt, J->pc)); | ||
405 | if (J->parent) { | ||
406 | setintV(L->top++, J->parent); | ||
407 | setintV(L->top++, J->exitno); | ||
408 | } | ||
409 | ); | ||
410 | lj_record_setup(J); | ||
411 | } | ||
412 | |||
413 | /* Stop tracing. */ | ||
414 | static void trace_stop(jit_State *J) | ||
415 | { | ||
416 | BCIns *pc = mref(J->cur.startpc, BCIns); | ||
417 | BCOp op = bc_op(J->cur.startins); | ||
418 | GCproto *pt = &gcref(J->cur.startpt)->pt; | ||
419 | TraceNo traceno = J->cur.traceno; | ||
420 | lua_State *L; | ||
421 | |||
422 | switch (op) { | ||
423 | case BC_FORL: | ||
424 | setbc_op(pc+bc_j(J->cur.startins), BC_JFORI); /* Patch FORI, too. */ | ||
425 | /* fallthrough */ | ||
426 | case BC_LOOP: | ||
427 | case BC_ITERL: | ||
428 | case BC_FUNCF: | ||
429 | /* Patch bytecode of starting instruction in root trace. */ | ||
430 | setbc_op(pc, (int)op+(int)BC_JLOOP-(int)BC_LOOP); | ||
431 | setbc_d(pc, traceno); | ||
432 | addroot: | ||
433 | /* Add to root trace chain in prototype. */ | ||
434 | J->cur.nextroot = pt->trace; | ||
435 | pt->trace = (TraceNo1)traceno; | ||
436 | break; | ||
437 | case BC_RET: | ||
438 | case BC_RET0: | ||
439 | case BC_RET1: | ||
440 | *pc = BCINS_AD(BC_JLOOP, J->cur.snap[0].nslots, traceno); | ||
441 | goto addroot; | ||
442 | case BC_JMP: | ||
443 | /* Patch exit branch in parent to side trace entry. */ | ||
444 | lua_assert(J->parent != 0 && J->cur.root != 0); | ||
445 | lj_asm_patchexit(J, traceref(J, J->parent), J->exitno, J->cur.mcode); | ||
446 | /* Avoid compiling a side trace twice (stack resizing uses parent exit). */ | ||
447 | traceref(J, J->parent)->snap[J->exitno].count = SNAPCOUNT_DONE; | ||
448 | /* Add to side trace chain in root trace. */ | ||
449 | { | ||
450 | GCtrace *root = traceref(J, J->cur.root); | ||
451 | root->nchild++; | ||
452 | J->cur.nextside = root->nextside; | ||
453 | root->nextside = (TraceNo1)traceno; | ||
454 | } | ||
455 | break; | ||
456 | default: | ||
457 | lua_assert(0); | ||
458 | break; | ||
459 | } | ||
460 | |||
461 | /* Commit new mcode only after all patching is done. */ | ||
462 | lj_mcode_commit(J, J->cur.mcode); | ||
463 | J->postproc = LJ_POST_NONE; | ||
464 | trace_save(J); | ||
465 | |||
466 | L = J->L; | ||
467 | lj_vmevent_send(L, TRACE, | ||
468 | setstrV(L, L->top++, lj_str_newlit(L, "stop")); | ||
469 | setintV(L->top++, traceno); | ||
470 | ); | ||
471 | } | ||
472 | |||
473 | /* Start a new root trace for down-recursion. */ | ||
474 | static int trace_downrec(jit_State *J) | ||
475 | { | ||
476 | /* Restart recording at the return instruction. */ | ||
477 | lua_assert(J->pt != NULL); | ||
478 | lua_assert(bc_isret(bc_op(*J->pc))); | ||
479 | if (bc_op(*J->pc) == BC_RETM) | ||
480 | return 0; /* NYI: down-recursion with RETM. */ | ||
481 | J->parent = 0; | ||
482 | J->exitno = 0; | ||
483 | J->state = LJ_TRACE_RECORD; | ||
484 | trace_start(J); | ||
485 | return 1; | ||
486 | } | ||
487 | |||
488 | /* Abort tracing. */ | ||
489 | static int trace_abort(jit_State *J) | ||
490 | { | ||
491 | lua_State *L = J->L; | ||
492 | TraceError e = LJ_TRERR_RECERR; | ||
493 | TraceNo traceno; | ||
494 | |||
495 | J->postproc = LJ_POST_NONE; | ||
496 | lj_mcode_abort(J); | ||
497 | if (tvisnumber(L->top-1)) | ||
498 | e = (TraceError)numberVint(L->top-1); | ||
499 | if (e == LJ_TRERR_MCODELM) { | ||
500 | L->top--; /* Remove error object */ | ||
501 | J->state = LJ_TRACE_ASM; | ||
502 | return 1; /* Retry ASM with new MCode area. */ | ||
503 | } | ||
504 | /* Penalize or blacklist starting bytecode instruction. */ | ||
505 | if (J->parent == 0 && !bc_isret(bc_op(J->cur.startins))) | ||
506 | penalty_pc(J, &gcref(J->cur.startpt)->pt, mref(J->cur.startpc, BCIns), e); | ||
507 | |||
508 | /* Is there anything to abort? */ | ||
509 | traceno = J->cur.traceno; | ||
510 | if (traceno) { | ||
511 | ptrdiff_t errobj = savestack(L, L->top-1); /* Stack may be resized. */ | ||
512 | J->cur.link = 0; | ||
513 | J->cur.linktype = LJ_TRLINK_NONE; | ||
514 | lj_vmevent_send(L, TRACE, | ||
515 | TValue *frame; | ||
516 | const BCIns *pc; | ||
517 | GCfunc *fn; | ||
518 | setstrV(L, L->top++, lj_str_newlit(L, "abort")); | ||
519 | setintV(L->top++, traceno); | ||
520 | /* Find original Lua function call to generate a better error message. */ | ||
521 | frame = J->L->base-1; | ||
522 | pc = J->pc; | ||
523 | while (!isluafunc(frame_func(frame))) { | ||
524 | pc = (frame_iscont(frame) ? frame_contpc(frame) : frame_pc(frame)) - 1; | ||
525 | frame = frame_prev(frame); | ||
526 | } | ||
527 | fn = frame_func(frame); | ||
528 | setfuncV(L, L->top++, fn); | ||
529 | setintV(L->top++, proto_bcpos(funcproto(fn), pc)); | ||
530 | copyTV(L, L->top++, restorestack(L, errobj)); | ||
531 | copyTV(L, L->top++, &J->errinfo); | ||
532 | ); | ||
533 | /* Drop aborted trace after the vmevent (which may still access it). */ | ||
534 | setgcrefnull(J->trace[traceno]); | ||
535 | if (traceno < J->freetrace) | ||
536 | J->freetrace = traceno; | ||
537 | J->cur.traceno = 0; | ||
538 | } | ||
539 | L->top--; /* Remove error object */ | ||
540 | if (e == LJ_TRERR_DOWNREC) | ||
541 | return trace_downrec(J); | ||
542 | else if (e == LJ_TRERR_MCODEAL) | ||
543 | lj_trace_flushall(L); | ||
544 | return 0; | ||
545 | } | ||
546 | |||
547 | /* Perform pending re-patch of a bytecode instruction. */ | ||
548 | static LJ_AINLINE void trace_pendpatch(jit_State *J, int force) | ||
549 | { | ||
550 | if (LJ_UNLIKELY(J->patchpc)) { | ||
551 | if (force || J->bcskip == 0) { | ||
552 | *J->patchpc = J->patchins; | ||
553 | J->patchpc = NULL; | ||
554 | } else { | ||
555 | J->bcskip = 0; | ||
556 | } | ||
557 | } | ||
558 | } | ||
559 | |||
560 | /* State machine for the trace compiler. Protected callback. */ | ||
561 | static TValue *trace_state(lua_State *L, lua_CFunction dummy, void *ud) | ||
562 | { | ||
563 | jit_State *J = (jit_State *)ud; | ||
564 | UNUSED(dummy); | ||
565 | do { | ||
566 | retry: | ||
567 | switch (J->state) { | ||
568 | case LJ_TRACE_START: | ||
569 | J->state = LJ_TRACE_RECORD; /* trace_start() may change state. */ | ||
570 | trace_start(J); | ||
571 | lj_dispatch_update(J2G(J)); | ||
572 | break; | ||
573 | |||
574 | case LJ_TRACE_RECORD: | ||
575 | trace_pendpatch(J, 0); | ||
576 | setvmstate(J2G(J), RECORD); | ||
577 | lj_vmevent_send_(L, RECORD, | ||
578 | /* Save/restore tmptv state for trace recorder. */ | ||
579 | TValue savetv = J2G(J)->tmptv; | ||
580 | TValue savetv2 = J2G(J)->tmptv2; | ||
581 | setintV(L->top++, J->cur.traceno); | ||
582 | setfuncV(L, L->top++, J->fn); | ||
583 | setintV(L->top++, J->pt ? (int32_t)proto_bcpos(J->pt, J->pc) : -1); | ||
584 | setintV(L->top++, J->framedepth); | ||
585 | , | ||
586 | J2G(J)->tmptv = savetv; | ||
587 | J2G(J)->tmptv2 = savetv2; | ||
588 | ); | ||
589 | lj_record_ins(J); | ||
590 | break; | ||
591 | |||
592 | case LJ_TRACE_END: | ||
593 | trace_pendpatch(J, 1); | ||
594 | J->loopref = 0; | ||
595 | if ((J->flags & JIT_F_OPT_LOOP) && | ||
596 | J->cur.link == J->cur.traceno && J->framedepth + J->retdepth == 0) { | ||
597 | setvmstate(J2G(J), OPT); | ||
598 | lj_opt_dce(J); | ||
599 | if (lj_opt_loop(J)) { /* Loop optimization failed? */ | ||
600 | J->cur.link = 0; | ||
601 | J->cur.linktype = LJ_TRLINK_NONE; | ||
602 | J->loopref = J->cur.nins; | ||
603 | J->state = LJ_TRACE_RECORD; /* Try to continue recording. */ | ||
604 | break; | ||
605 | } | ||
606 | J->loopref = J->chain[IR_LOOP]; /* Needed by assembler. */ | ||
607 | } | ||
608 | lj_opt_split(J); | ||
609 | J->state = LJ_TRACE_ASM; | ||
610 | break; | ||
611 | |||
612 | case LJ_TRACE_ASM: | ||
613 | setvmstate(J2G(J), ASM); | ||
614 | lj_asm_trace(J, &J->cur); | ||
615 | trace_stop(J); | ||
616 | setvmstate(J2G(J), INTERP); | ||
617 | J->state = LJ_TRACE_IDLE; | ||
618 | lj_dispatch_update(J2G(J)); | ||
619 | return NULL; | ||
620 | |||
621 | default: /* Trace aborted asynchronously. */ | ||
622 | setintV(L->top++, (int32_t)LJ_TRERR_RECERR); | ||
623 | /* fallthrough */ | ||
624 | case LJ_TRACE_ERR: | ||
625 | trace_pendpatch(J, 1); | ||
626 | if (trace_abort(J)) | ||
627 | goto retry; | ||
628 | setvmstate(J2G(J), INTERP); | ||
629 | J->state = LJ_TRACE_IDLE; | ||
630 | lj_dispatch_update(J2G(J)); | ||
631 | return NULL; | ||
632 | } | ||
633 | } while (J->state > LJ_TRACE_RECORD); | ||
634 | return NULL; | ||
635 | } | ||
636 | |||
637 | /* -- Event handling ------------------------------------------------------ */ | ||
638 | |||
639 | /* A bytecode instruction is about to be executed. Record it. */ | ||
640 | void lj_trace_ins(jit_State *J, const BCIns *pc) | ||
641 | { | ||
642 | /* Note: J->L must already be set. pc is the true bytecode PC here. */ | ||
643 | J->pc = pc; | ||
644 | J->fn = curr_func(J->L); | ||
645 | J->pt = isluafunc(J->fn) ? funcproto(J->fn) : NULL; | ||
646 | while (lj_vm_cpcall(J->L, NULL, (void *)J, trace_state) != 0) | ||
647 | J->state = LJ_TRACE_ERR; | ||
648 | } | ||
649 | |||
650 | /* A hotcount triggered. Start recording a root trace. */ | ||
651 | void LJ_FASTCALL lj_trace_hot(jit_State *J, const BCIns *pc) | ||
652 | { | ||
653 | /* Note: pc is the interpreter bytecode PC here. It's offset by 1. */ | ||
654 | ERRNO_SAVE | ||
655 | /* Reset hotcount. */ | ||
656 | hotcount_set(J2GG(J), pc, J->param[JIT_P_hotloop]*HOTCOUNT_LOOP); | ||
657 | /* Only start a new trace if not recording or inside __gc call or vmevent. */ | ||
658 | if (J->state == LJ_TRACE_IDLE && | ||
659 | !(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT))) { | ||
660 | J->parent = 0; /* Root trace. */ | ||
661 | J->exitno = 0; | ||
662 | J->state = LJ_TRACE_START; | ||
663 | lj_trace_ins(J, pc-1); | ||
664 | } | ||
665 | ERRNO_RESTORE | ||
666 | } | ||
667 | |||
668 | /* Check for a hot side exit. If yes, start recording a side trace. */ | ||
669 | static void trace_hotside(jit_State *J, const BCIns *pc) | ||
670 | { | ||
671 | SnapShot *snap = &traceref(J, J->parent)->snap[J->exitno]; | ||
672 | if (!(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT)) && | ||
673 | snap->count != SNAPCOUNT_DONE && | ||
674 | ++snap->count >= J->param[JIT_P_hotexit]) { | ||
675 | lua_assert(J->state == LJ_TRACE_IDLE); | ||
676 | /* J->parent is non-zero for a side trace. */ | ||
677 | J->state = LJ_TRACE_START; | ||
678 | lj_trace_ins(J, pc); | ||
679 | } | ||
680 | } | ||
681 | |||
682 | /* Tiny struct to pass data to protected call. */ | ||
683 | typedef struct ExitDataCP { | ||
684 | jit_State *J; | ||
685 | void *exptr; /* Pointer to exit state. */ | ||
686 | const BCIns *pc; /* Restart interpreter at this PC. */ | ||
687 | } ExitDataCP; | ||
688 | |||
689 | /* Need to protect lj_snap_restore because it may throw. */ | ||
690 | static TValue *trace_exit_cp(lua_State *L, lua_CFunction dummy, void *ud) | ||
691 | { | ||
692 | ExitDataCP *exd = (ExitDataCP *)ud; | ||
693 | cframe_errfunc(L->cframe) = -1; /* Inherit error function. */ | ||
694 | exd->pc = lj_snap_restore(exd->J, exd->exptr); | ||
695 | UNUSED(dummy); | ||
696 | return NULL; | ||
697 | } | ||
698 | |||
699 | #ifndef LUAJIT_DISABLE_VMEVENT | ||
700 | /* Push all registers from exit state. */ | ||
701 | static void trace_exit_regs(lua_State *L, ExitState *ex) | ||
702 | { | ||
703 | int32_t i; | ||
704 | setintV(L->top++, RID_NUM_GPR); | ||
705 | setintV(L->top++, RID_NUM_FPR); | ||
706 | for (i = 0; i < RID_NUM_GPR; i++) { | ||
707 | if (sizeof(ex->gpr[i]) == sizeof(int32_t)) | ||
708 | setintV(L->top++, (int32_t)ex->gpr[i]); | ||
709 | else | ||
710 | setnumV(L->top++, (lua_Number)ex->gpr[i]); | ||
711 | } | ||
712 | #if !LJ_SOFTFP | ||
713 | for (i = 0; i < RID_NUM_FPR; i++) { | ||
714 | setnumV(L->top, ex->fpr[i]); | ||
715 | if (LJ_UNLIKELY(tvisnan(L->top))) | ||
716 | setnanV(L->top); | ||
717 | L->top++; | ||
718 | } | ||
719 | #endif | ||
720 | } | ||
721 | #endif | ||
722 | |||
723 | #ifdef EXITSTATE_PCREG | ||
724 | /* Determine trace number from pc of exit instruction. */ | ||
725 | static TraceNo trace_exit_find(jit_State *J, MCode *pc) | ||
726 | { | ||
727 | TraceNo traceno; | ||
728 | for (traceno = 1; traceno < J->sizetrace; traceno++) { | ||
729 | GCtrace *T = traceref(J, traceno); | ||
730 | if (T && pc >= T->mcode && pc < (MCode *)((char *)T->mcode + T->szmcode)) | ||
731 | return traceno; | ||
732 | } | ||
733 | lua_assert(0); | ||
734 | return 0; | ||
735 | } | ||
736 | #endif | ||
737 | |||
738 | /* A trace exited. Restore interpreter state. */ | ||
739 | int LJ_FASTCALL lj_trace_exit(jit_State *J, void *exptr) | ||
740 | { | ||
741 | ERRNO_SAVE | ||
742 | lua_State *L = J->L; | ||
743 | ExitState *ex = (ExitState *)exptr; | ||
744 | ExitDataCP exd; | ||
745 | int errcode; | ||
746 | const BCIns *pc; | ||
747 | void *cf; | ||
748 | GCtrace *T; | ||
749 | #ifdef EXITSTATE_PCREG | ||
750 | J->parent = trace_exit_find(J, (MCode *)(intptr_t)ex->gpr[EXITSTATE_PCREG]); | ||
751 | #endif | ||
752 | T = traceref(J, J->parent); UNUSED(T); | ||
753 | #ifdef EXITSTATE_CHECKEXIT | ||
754 | if (J->exitno == T->nsnap) { /* Treat stack check like a parent exit. */ | ||
755 | lua_assert(T->root != 0); | ||
756 | J->exitno = T->ir[REF_BASE].op2; | ||
757 | J->parent = T->ir[REF_BASE].op1; | ||
758 | T = traceref(J, J->parent); | ||
759 | } | ||
760 | #endif | ||
761 | lua_assert(T != NULL && J->exitno < T->nsnap); | ||
762 | exd.J = J; | ||
763 | exd.exptr = exptr; | ||
764 | errcode = lj_vm_cpcall(L, NULL, &exd, trace_exit_cp); | ||
765 | if (errcode) | ||
766 | return -errcode; /* Return negated error code. */ | ||
767 | |||
768 | lj_vmevent_send(L, TEXIT, | ||
769 | lj_state_checkstack(L, 4+RID_NUM_GPR+RID_NUM_FPR+LUA_MINSTACK); | ||
770 | setintV(L->top++, J->parent); | ||
771 | setintV(L->top++, J->exitno); | ||
772 | trace_exit_regs(L, ex); | ||
773 | ); | ||
774 | |||
775 | pc = exd.pc; | ||
776 | cf = cframe_raw(L->cframe); | ||
777 | setcframe_pc(cf, pc); | ||
778 | if (G(L)->gc.state == GCSatomic || G(L)->gc.state == GCSfinalize) | ||
779 | lj_gc_step(L); /* Exited because of GC: drive GC forward. */ | ||
780 | else | ||
781 | trace_hotside(J, pc); | ||
782 | if (bc_op(*pc) == BC_JLOOP) { | ||
783 | BCIns *retpc = &traceref(J, bc_d(*pc))->startins; | ||
784 | if (bc_isret(bc_op(*retpc))) { | ||
785 | if (J->state == LJ_TRACE_RECORD) { | ||
786 | J->patchins = *pc; | ||
787 | J->patchpc = (BCIns *)pc; | ||
788 | *J->patchpc = *retpc; | ||
789 | J->bcskip = 1; | ||
790 | } else { | ||
791 | pc = retpc; | ||
792 | setcframe_pc(cf, pc); | ||
793 | } | ||
794 | } | ||
795 | } | ||
796 | /* Return MULTRES or 0. */ | ||
797 | ERRNO_RESTORE | ||
798 | switch (bc_op(*pc)) { | ||
799 | case BC_CALLM: case BC_CALLMT: | ||
800 | return (int)((BCReg)(L->top - L->base) - bc_a(*pc) - bc_c(*pc)); | ||
801 | case BC_RETM: | ||
802 | return (int)((BCReg)(L->top - L->base) + 1 - bc_a(*pc) - bc_d(*pc)); | ||
803 | case BC_TSETM: | ||
804 | return (int)((BCReg)(L->top - L->base) + 1 - bc_a(*pc)); | ||
805 | default: | ||
806 | if (bc_op(*pc) >= BC_FUNCF) | ||
807 | return (int)((BCReg)(L->top - L->base) + 1); | ||
808 | return 0; | ||
809 | } | ||
810 | } | ||
811 | |||
812 | #endif | ||