From 6523585c66c04cea54df50013df8886b589847d8 Mon Sep 17 00:00:00 2001 From: David Walter Seikel Date: Mon, 23 Jan 2012 23:36:30 +1000 Subject: Add luaproc and LuaJIT libraries. Two versions of LuaJIT, the stable release, and the dev version. Try the dev version first, until ih fails badly. --- libraries/luajit-2.0/src/lj_record.c | 2228 ++++++++++++++++++++++++++++++++++ 1 file changed, 2228 insertions(+) create mode 100644 libraries/luajit-2.0/src/lj_record.c (limited to 'libraries/luajit-2.0/src/lj_record.c') diff --git a/libraries/luajit-2.0/src/lj_record.c b/libraries/luajit-2.0/src/lj_record.c new file mode 100644 index 0000000..2c27a71 --- /dev/null +++ b/libraries/luajit-2.0/src/lj_record.c @@ -0,0 +1,2228 @@ +/* +** Trace recorder (bytecode -> SSA IR). +** Copyright (C) 2005-2011 Mike Pall. See Copyright Notice in luajit.h +*/ + +#define lj_record_c +#define LUA_CORE + +#include "lj_obj.h" + +#if LJ_HASJIT + +#include "lj_err.h" +#include "lj_str.h" +#include "lj_tab.h" +#include "lj_meta.h" +#include "lj_frame.h" +#include "lj_bc.h" +#include "lj_ff.h" +#include "lj_ir.h" +#include "lj_jit.h" +#include "lj_ircall.h" +#include "lj_iropt.h" +#include "lj_trace.h" +#include "lj_record.h" +#include "lj_ffrecord.h" +#include "lj_snap.h" +#include "lj_dispatch.h" +#include "lj_vm.h" + +/* Some local macros to save typing. Undef'd at the end. */ +#define IR(ref) (&J->cur.ir[(ref)]) + +/* Pass IR on to next optimization in chain (FOLD). */ +#define emitir(ot, a, b) (lj_ir_set(J, (ot), (a), (b)), lj_opt_fold(J)) + +/* Emit raw IR without passing through optimizations. */ +#define emitir_raw(ot, a, b) (lj_ir_set(J, (ot), (a), (b)), lj_ir_emit(J)) + +/* -- Sanity checks ------------------------------------------------------- */ + +#ifdef LUA_USE_ASSERT +/* Sanity check the whole IR -- sloooow. */ +static void rec_check_ir(jit_State *J) +{ + IRRef i, nins = J->cur.nins, nk = J->cur.nk; + lua_assert(nk <= REF_BIAS && nins >= REF_BIAS && nins < 65536); + for (i = nins-1; i >= nk; i--) { + IRIns *ir = IR(i); + uint32_t mode = lj_ir_mode[ir->o]; + IRRef op1 = ir->op1; + IRRef op2 = ir->op2; + switch (irm_op1(mode)) { + case IRMnone: lua_assert(op1 == 0); break; + case IRMref: lua_assert(op1 >= nk); + lua_assert(i >= REF_BIAS ? op1 < i : op1 > i); break; + case IRMlit: break; + case IRMcst: lua_assert(i < REF_BIAS); continue; + } + switch (irm_op2(mode)) { + case IRMnone: lua_assert(op2 == 0); break; + case IRMref: lua_assert(op2 >= nk); + lua_assert(i >= REF_BIAS ? op2 < i : op2 > i); break; + case IRMlit: break; + case IRMcst: lua_assert(0); break; + } + if (ir->prev) { + lua_assert(ir->prev >= nk); + lua_assert(i >= REF_BIAS ? ir->prev < i : ir->prev > i); + lua_assert(ir->o == IR_NOP || IR(ir->prev)->o == ir->o); + } + } +} + +/* Compare stack slots and frames of the recorder and the VM. */ +static void rec_check_slots(jit_State *J) +{ + BCReg s, nslots = J->baseslot + J->maxslot; + int32_t depth = 0; + cTValue *base = J->L->base - J->baseslot; + lua_assert(J->baseslot >= 1 && J->baseslot < LJ_MAX_JSLOTS); + lua_assert(J->baseslot == 1 || (J->slot[J->baseslot-1] & TREF_FRAME)); + lua_assert(nslots < LJ_MAX_JSLOTS); + for (s = 0; s < nslots; s++) { + TRef tr = J->slot[s]; + if (tr) { + cTValue *tv = &base[s]; + IRRef ref = tref_ref(tr); + IRIns *ir; + lua_assert(ref >= J->cur.nk && ref < J->cur.nins); + ir = IR(ref); + lua_assert(irt_t(ir->t) == tref_t(tr)); + if (s == 0) { + lua_assert(tref_isfunc(tr)); + } else if ((tr & TREF_FRAME)) { + GCfunc *fn = gco2func(frame_gc(tv)); + BCReg delta = (BCReg)(tv - frame_prev(tv)); + lua_assert(tref_isfunc(tr)); + if (tref_isk(tr)) lua_assert(fn == ir_kfunc(ir)); + lua_assert(s > delta ? (J->slot[s-delta] & TREF_FRAME) : (s == delta)); + depth++; + } else if ((tr & TREF_CONT)) { + lua_assert(ir_kptr(ir) == gcrefp(tv->gcr, void)); + lua_assert((J->slot[s+1] & TREF_FRAME)); + depth++; + } else { + if (tvisnumber(tv)) + lua_assert(tref_isnumber(tr)); /* Could be IRT_INT etc., too. */ + else + lua_assert(itype2irt(tv) == tref_type(tr)); + if (tref_isk(tr)) { /* Compare constants. */ + TValue tvk; + lj_ir_kvalue(J->L, &tvk, ir); + if (!(tvisnum(&tvk) && tvisnan(&tvk))) + lua_assert(lj_obj_equal(tv, &tvk)); + else + lua_assert(tvisnum(tv) && tvisnan(tv)); + } + } + } + } + lua_assert(J->framedepth == depth); +} +#endif + +/* -- Type handling and specialization ------------------------------------ */ + +/* Note: these functions return tagged references (TRef). */ + +/* Specialize a slot to a specific type. Note: slot can be negative! */ +static TRef sloadt(jit_State *J, int32_t slot, IRType t, int mode) +{ + /* Caller may set IRT_GUARD in t. */ + TRef ref = emitir_raw(IRT(IR_SLOAD, t), (int32_t)J->baseslot+slot, mode); + J->base[slot] = ref; + return ref; +} + +/* Specialize a slot to the runtime type. Note: slot can be negative! */ +static TRef sload(jit_State *J, int32_t slot) +{ + IRType t = itype2irt(&J->L->base[slot]); + TRef ref = emitir_raw(IRTG(IR_SLOAD, t), (int32_t)J->baseslot+slot, + IRSLOAD_TYPECHECK); + if (irtype_ispri(t)) ref = TREF_PRI(t); /* Canonicalize primitive refs. */ + J->base[slot] = ref; + return ref; +} + +/* Get TRef from slot. Load slot and specialize if not done already. */ +#define getslot(J, s) (J->base[(s)] ? J->base[(s)] : sload(J, (int32_t)(s))) + +/* Get TRef for current function. */ +static TRef getcurrf(jit_State *J) +{ + if (J->base[-1]) + return J->base[-1]; + lua_assert(J->baseslot == 1); + return sloadt(J, -1, IRT_FUNC, IRSLOAD_READONLY); +} + +/* Compare for raw object equality. +** Returns 0 if the objects are the same. +** Returns 1 if they are different, but the same type. +** Returns 2 for two different types. +** Comparisons between primitives always return 1 -- no caller cares about it. +*/ +int lj_record_objcmp(jit_State *J, TRef a, TRef b, cTValue *av, cTValue *bv) +{ + int diff = !lj_obj_equal(av, bv); + if (!tref_isk2(a, b)) { /* Shortcut, also handles primitives. */ + IRType ta = tref_isinteger(a) ? IRT_INT : tref_type(a); + IRType tb = tref_isinteger(b) ? IRT_INT : tref_type(b); + if (ta != tb) { + /* Widen mixed number/int comparisons to number/number comparison. */ + if (ta == IRT_INT && tb == IRT_NUM) { + a = emitir(IRTN(IR_CONV), a, IRCONV_NUM_INT); + ta = IRT_NUM; + } else if (ta == IRT_NUM && tb == IRT_INT) { + b = emitir(IRTN(IR_CONV), b, IRCONV_NUM_INT); + } else { + return 2; /* Two different types are never equal. */ + } + } + emitir(IRTG(diff ? IR_NE : IR_EQ, ta), a, b); + } + return diff; +} + +/* -- Record loop ops ----------------------------------------------------- */ + +/* Loop event. */ +typedef enum { + LOOPEV_LEAVE, /* Loop is left or not entered. */ + LOOPEV_ENTERLO, /* Loop is entered with a low iteration count left. */ + LOOPEV_ENTER /* Loop is entered. */ +} LoopEvent; + +/* Canonicalize slots: convert integers to numbers. */ +static void canonicalize_slots(jit_State *J) +{ + BCReg s; + if (LJ_DUALNUM) return; + for (s = J->baseslot+J->maxslot-1; s >= 1; s--) { + TRef tr = J->slot[s]; + if (tref_isinteger(tr)) { + IRIns *ir = IR(tref_ref(tr)); + if (!(ir->o == IR_SLOAD && (ir->op2 & IRSLOAD_READONLY))) + J->slot[s] = emitir(IRTN(IR_CONV), tr, IRCONV_NUM_INT); + } + } +} + +/* Stop recording. */ +static void rec_stop(jit_State *J, TraceLink linktype, TraceNo lnk) +{ + lj_trace_end(J); + J->cur.linktype = (uint8_t)linktype; + J->cur.link = (uint16_t)lnk; + /* Looping back at the same stack level? */ + if (lnk == J->cur.traceno && J->framedepth + J->retdepth == 0) { + if ((J->flags & JIT_F_OPT_LOOP)) /* Shall we try to create a loop? */ + goto nocanon; /* Do not canonicalize or we lose the narrowing. */ + if (J->cur.root) /* Otherwise ensure we always link to the root trace. */ + J->cur.link = J->cur.root; + } + canonicalize_slots(J); +nocanon: + /* Note: all loop ops must set J->pc to the following instruction! */ + lj_snap_add(J); /* Add loop snapshot. */ + J->needsnap = 0; + J->mergesnap = 1; /* In case recording continues. */ +} + +/* Search bytecode backwards for a int/num constant slot initializer. */ +static TRef find_kinit(jit_State *J, const BCIns *endpc, BCReg slot, IRType t) +{ + /* This algorithm is rather simplistic and assumes quite a bit about + ** how the bytecode is generated. It works fine for FORI initializers, + ** but it won't necessarily work in other cases (e.g. iterator arguments). + ** It doesn't do anything fancy, either (like backpropagating MOVs). + */ + const BCIns *pc, *startpc = proto_bc(J->pt); + for (pc = endpc-1; pc > startpc; pc--) { + BCIns ins = *pc; + BCOp op = bc_op(ins); + /* First try to find the last instruction that stores to this slot. */ + if (bcmode_a(op) == BCMbase && bc_a(ins) <= slot) { + return 0; /* Multiple results, e.g. from a CALL or KNIL. */ + } else if (bcmode_a(op) == BCMdst && bc_a(ins) == slot) { + if (op == BC_KSHORT || op == BC_KNUM) { /* Found const. initializer. */ + /* Now try to verify there's no forward jump across it. */ + const BCIns *kpc = pc; + for (; pc > startpc; pc--) + if (bc_op(*pc) == BC_JMP) { + const BCIns *target = pc+bc_j(*pc)+1; + if (target > kpc && target <= endpc) + return 0; /* Conditional assignment. */ + } + if (op == BC_KSHORT) { + int32_t k = (int32_t)(int16_t)bc_d(ins); + return t == IRT_INT ? lj_ir_kint(J, k) : lj_ir_knum(J, (lua_Number)k); + } else { + cTValue *tv = proto_knumtv(J->pt, bc_d(ins)); + if (t == IRT_INT) { + int32_t k = numberVint(tv); + if (tvisint(tv) || numV(tv) == (lua_Number)k) /* -0 is ok here. */ + return lj_ir_kint(J, k); + return 0; /* Type mismatch. */ + } else { + return lj_ir_knum(J, numberVnum(tv)); + } + } + } + return 0; /* Non-constant initializer. */ + } + } + return 0; /* No assignment to this slot found? */ +} + +/* Load and optionally convert a FORI argument from a slot. */ +static TRef fori_load(jit_State *J, BCReg slot, IRType t, int mode) +{ + int conv = (tvisint(&J->L->base[slot]) != (t==IRT_INT)) ? IRSLOAD_CONVERT : 0; + return sloadt(J, (int32_t)slot, + t + (((mode & IRSLOAD_TYPECHECK) || + (conv && t == IRT_INT && !(mode >> 16))) ? + IRT_GUARD : 0), + mode + conv); +} + +/* Peek before FORI to find a const initializer. Otherwise load from slot. */ +static TRef fori_arg(jit_State *J, const BCIns *fori, BCReg slot, + IRType t, int mode) +{ + TRef tr = J->base[slot]; + if (!tr) { + tr = find_kinit(J, fori, slot, t); + if (!tr) + tr = fori_load(J, slot, t, mode); + } + return tr; +} + +/* Return the direction of the FOR loop iterator. +** It's important to exactly reproduce the semantics of the interpreter. +*/ +static int rec_for_direction(cTValue *o) +{ + return (tvisint(o) ? intV(o) : (int32_t)o->u32.hi) >= 0; +} + +/* Simulate the runtime behavior of the FOR loop iterator. */ +static LoopEvent rec_for_iter(IROp *op, cTValue *o, int isforl) +{ + lua_Number stopv = numberVnum(&o[FORL_STOP]); + lua_Number idxv = numberVnum(&o[FORL_IDX]); + lua_Number stepv = numberVnum(&o[FORL_STEP]); + if (isforl) + idxv += stepv; + if (rec_for_direction(&o[FORL_STEP])) { + if (idxv <= stopv) { + *op = IR_LE; + return idxv + 2*stepv > stopv ? LOOPEV_ENTERLO : LOOPEV_ENTER; + } + *op = IR_GT; return LOOPEV_LEAVE; + } else { + if (stopv <= idxv) { + *op = IR_GE; + return idxv + 2*stepv < stopv ? LOOPEV_ENTERLO : LOOPEV_ENTER; + } + *op = IR_LT; return LOOPEV_LEAVE; + } +} + +/* Record checks for FOR loop overflow and step direction. */ +static void rec_for_check(jit_State *J, IRType t, int dir, + TRef stop, TRef step, int init) +{ + if (!tref_isk(step)) { + /* Non-constant step: need a guard for the direction. */ + TRef zero = (t == IRT_INT) ? lj_ir_kint(J, 0) : lj_ir_knum_zero(J); + emitir(IRTG(dir ? IR_GE : IR_LT, t), step, zero); + /* Add hoistable overflow checks for a narrowed FORL index. */ + if (init && t == IRT_INT) { + if (tref_isk(stop)) { + /* Constant stop: optimize check away or to a range check for step. */ + int32_t k = IR(tref_ref(stop))->i; + if (dir) { + if (k > 0) + emitir(IRTGI(IR_LE), step, lj_ir_kint(J, (int32_t)0x7fffffff-k)); + } else { + if (k < 0) + emitir(IRTGI(IR_GE), step, lj_ir_kint(J, (int32_t)0x80000000-k)); + } + } else { + /* Stop+step variable: need full overflow check. */ + TRef tr = emitir(IRTGI(IR_ADDOV), step, stop); + emitir(IRTI(IR_USE), tr, 0); /* ADDOV is weak. Avoid dead result. */ + } + } + } else if (init && t == IRT_INT && !tref_isk(stop)) { + /* Constant step: optimize overflow check to a range check for stop. */ + int32_t k = IR(tref_ref(step))->i; + k = (int32_t)(dir ? 0x7fffffff : 0x80000000) - k; + emitir(IRTGI(dir ? IR_LE : IR_GE), stop, lj_ir_kint(J, k)); + } +} + +/* Record a FORL instruction. */ +static void rec_for_loop(jit_State *J, const BCIns *fori, ScEvEntry *scev, + int init) +{ + BCReg ra = bc_a(*fori); + cTValue *tv = &J->L->base[ra]; + TRef idx = J->base[ra+FORL_IDX]; + IRType t = idx ? tref_type(idx) : + (init || LJ_DUALNUM) ? lj_opt_narrow_forl(J, tv) : IRT_NUM; + int mode = IRSLOAD_INHERIT + + ((!LJ_DUALNUM || tvisint(tv) == (t == IRT_INT)) ? IRSLOAD_READONLY : 0); + TRef stop = fori_arg(J, fori, ra+FORL_STOP, t, mode); + TRef step = fori_arg(J, fori, ra+FORL_STEP, t, mode); + int tc, dir = rec_for_direction(&tv[FORL_STEP]); + lua_assert(bc_op(*fori) == BC_FORI || bc_op(*fori) == BC_JFORI); + scev->t.irt = t; + scev->dir = dir; + scev->stop = tref_ref(stop); + scev->step = tref_ref(step); + rec_for_check(J, t, dir, stop, step, init); + scev->start = tref_ref(find_kinit(J, fori, ra+FORL_IDX, IRT_INT)); + tc = (LJ_DUALNUM && + !(scev->start && irref_isk(scev->stop) && irref_isk(scev->step) && + tvisint(&tv[FORL_IDX]) == (t == IRT_INT))) ? + IRSLOAD_TYPECHECK : 0; + if (tc) { + J->base[ra+FORL_STOP] = stop; + J->base[ra+FORL_STEP] = step; + } + if (!idx) + idx = fori_load(J, ra+FORL_IDX, t, + IRSLOAD_INHERIT + tc + (J->scev.start << 16)); + if (!init) + J->base[ra+FORL_IDX] = idx = emitir(IRT(IR_ADD, t), idx, step); + J->base[ra+FORL_EXT] = idx; + scev->idx = tref_ref(idx); + J->maxslot = ra+FORL_EXT+1; +} + +/* Record FORL/JFORL or FORI/JFORI. */ +static LoopEvent rec_for(jit_State *J, const BCIns *fori, int isforl) +{ + BCReg ra = bc_a(*fori); + TValue *tv = &J->L->base[ra]; + TRef *tr = &J->base[ra]; + IROp op; + LoopEvent ev; + TRef stop; + IRType t; + if (isforl) { /* Handle FORL/JFORL opcodes. */ + TRef idx = tr[FORL_IDX]; + if (tref_ref(idx) == J->scev.idx) { + t = J->scev.t.irt; + stop = J->scev.stop; + idx = emitir(IRT(IR_ADD, t), idx, J->scev.step); + tr[FORL_EXT] = tr[FORL_IDX] = idx; + } else { + ScEvEntry scev; + rec_for_loop(J, fori, &scev, 0); + t = scev.t.irt; + stop = scev.stop; + } + } else { /* Handle FORI/JFORI opcodes. */ + BCReg i; + lj_meta_for(J->L, tv); + t = (LJ_DUALNUM || tref_isint(tr[FORL_IDX])) ? lj_opt_narrow_forl(J, tv) : + IRT_NUM; + for (i = FORL_IDX; i <= FORL_STEP; i++) { + if (!tr[i]) sload(J, ra+i); + lua_assert(tref_isnumber_str(tr[i])); + if (tref_isstr(tr[i])) + tr[i] = emitir(IRTG(IR_STRTO, IRT_NUM), tr[i], 0); + if (t == IRT_INT) { + if (!tref_isinteger(tr[i])) + tr[i] = emitir(IRTGI(IR_CONV), tr[i], IRCONV_INT_NUM|IRCONV_CHECK); + } else { + if (!tref_isnum(tr[i])) + tr[i] = emitir(IRTN(IR_CONV), tr[i], IRCONV_NUM_INT); + } + } + tr[FORL_EXT] = tr[FORL_IDX]; + stop = tr[FORL_STOP]; + rec_for_check(J, t, rec_for_direction(&tv[FORL_STEP]), + stop, tr[FORL_STEP], 1); + } + + ev = rec_for_iter(&op, tv, isforl); + if (ev == LOOPEV_LEAVE) { + J->maxslot = ra+FORL_EXT+1; + J->pc = fori+1; + } else { + J->maxslot = ra; + J->pc = fori+bc_j(*fori)+1; + } + lj_snap_add(J); + + emitir(IRTG(op, t), tr[FORL_IDX], stop); + + if (ev == LOOPEV_LEAVE) { + J->maxslot = ra; + J->pc = fori+bc_j(*fori)+1; + } else { + J->maxslot = ra+FORL_EXT+1; + J->pc = fori+1; + } + J->needsnap = 1; + return ev; +} + +/* Record ITERL/JITERL. */ +static LoopEvent rec_iterl(jit_State *J, const BCIns iterins) +{ + BCReg ra = bc_a(iterins); + lua_assert(J->base[ra] != 0); + if (!tref_isnil(J->base[ra])) { /* Looping back? */ + J->base[ra-1] = J->base[ra]; /* Copy result of ITERC to control var. */ + J->maxslot = ra-1+bc_b(J->pc[-1]); + J->pc += bc_j(iterins)+1; + return LOOPEV_ENTER; + } else { + J->maxslot = ra-3; + J->pc++; + return LOOPEV_LEAVE; + } +} + +/* Record LOOP/JLOOP. Now, that was easy. */ +static LoopEvent rec_loop(jit_State *J, BCReg ra) +{ + if (ra < J->maxslot) J->maxslot = ra; + J->pc++; + return LOOPEV_ENTER; +} + +/* Check if a loop repeatedly failed to trace because it didn't loop back. */ +static int innerloopleft(jit_State *J, const BCIns *pc) +{ + ptrdiff_t i; + for (i = 0; i < PENALTY_SLOTS; i++) + if (mref(J->penalty[i].pc, const BCIns) == pc) { + if ((J->penalty[i].reason == LJ_TRERR_LLEAVE || + J->penalty[i].reason == LJ_TRERR_LINNER) && + J->penalty[i].val >= 2*PENALTY_MIN) + return 1; + break; + } + return 0; +} + +/* Handle the case when an interpreted loop op is hit. */ +static void rec_loop_interp(jit_State *J, const BCIns *pc, LoopEvent ev) +{ + if (J->parent == 0) { + if (pc == J->startpc && J->framedepth + J->retdepth == 0) { + /* Same loop? */ + if (ev == LOOPEV_LEAVE) /* Must loop back to form a root trace. */ + lj_trace_err(J, LJ_TRERR_LLEAVE); + rec_stop(J, LJ_TRLINK_LOOP, J->cur.traceno); /* Looping root trace. */ + } else if (ev != LOOPEV_LEAVE) { /* Entering inner loop? */ + /* It's usually better to abort here and wait until the inner loop + ** is traced. But if the inner loop repeatedly didn't loop back, + ** this indicates a low trip count. In this case try unrolling + ** an inner loop even in a root trace. But it's better to be a bit + ** more conservative here and only do it for very short loops. + */ + if (!innerloopleft(J, pc)) + lj_trace_err(J, LJ_TRERR_LINNER); /* Root trace hit an inner loop. */ + if ((ev != LOOPEV_ENTERLO && + J->loopref && J->cur.nins - J->loopref > 24) || --J->loopunroll < 0) + lj_trace_err(J, LJ_TRERR_LUNROLL); /* Limit loop unrolling. */ + J->loopref = J->cur.nins; + } + } else if (ev != LOOPEV_LEAVE) { /* Side trace enters an inner loop. */ + J->loopref = J->cur.nins; + if (--J->loopunroll < 0) + lj_trace_err(J, LJ_TRERR_LUNROLL); /* Limit loop unrolling. */ + } /* Side trace continues across a loop that's left or not entered. */ +} + +/* Handle the case when an already compiled loop op is hit. */ +static void rec_loop_jit(jit_State *J, TraceNo lnk, LoopEvent ev) +{ + if (J->parent == 0) { /* Root trace hit an inner loop. */ + /* Better let the inner loop spawn a side trace back here. */ + lj_trace_err(J, LJ_TRERR_LINNER); + } else if (ev != LOOPEV_LEAVE) { /* Side trace enters a compiled loop. */ + J->instunroll = 0; /* Cannot continue across a compiled loop op. */ + if (J->pc == J->startpc && J->framedepth + J->retdepth == 0) + rec_stop(J, LJ_TRLINK_LOOP, J->cur.traceno); /* Form an extra loop. */ + else + rec_stop(J, LJ_TRLINK_ROOT, lnk); /* Link to the loop. */ + } /* Side trace continues across a loop that's left or not entered. */ +} + +/* -- Record calls and returns -------------------------------------------- */ + +/* Specialize to the runtime value of the called function or its prototype. */ +static TRef rec_call_specialize(jit_State *J, GCfunc *fn, TRef tr) +{ + TRef kfunc; + if (isluafunc(fn)) { + GCproto *pt = funcproto(fn); + /* 3 or more closures created? Probably not a monomorphic function. */ + if (pt->flags >= 3*PROTO_CLCOUNT) { /* Specialize to prototype instead. */ + TRef trpt = emitir(IRT(IR_FLOAD, IRT_P32), tr, IRFL_FUNC_PC); + emitir(IRTG(IR_EQ, IRT_P32), trpt, lj_ir_kptr(J, proto_bc(pt))); + (void)lj_ir_kgc(J, obj2gco(pt), IRT_PROTO); /* Prevent GC of proto. */ + return tr; + } + } + /* Otherwise specialize to the function (closure) value itself. */ + kfunc = lj_ir_kfunc(J, fn); + emitir(IRTG(IR_EQ, IRT_FUNC), tr, kfunc); + return kfunc; +} + +/* Record call setup. */ +static void rec_call_setup(jit_State *J, BCReg func, ptrdiff_t nargs) +{ + RecordIndex ix; + TValue *functv = &J->L->base[func]; + TRef *fbase = &J->base[func]; + ptrdiff_t i; + for (i = 0; i <= nargs; i++) + (void)getslot(J, func+i); /* Ensure func and all args have a reference. */ + if (!tref_isfunc(fbase[0])) { /* Resolve __call metamethod. */ + ix.tab = fbase[0]; + copyTV(J->L, &ix.tabv, functv); + if (!lj_record_mm_lookup(J, &ix, MM_call) || !tref_isfunc(ix.mobj)) + lj_trace_err(J, LJ_TRERR_NOMM); + for (i = ++nargs; i > 0; i--) /* Shift arguments up. */ + fbase[i] = fbase[i-1]; + fbase[0] = ix.mobj; /* Replace function. */ + functv = &ix.mobjv; + } + fbase[0] = TREF_FRAME | rec_call_specialize(J, funcV(functv), fbase[0]); + J->maxslot = (BCReg)nargs; +} + +/* Record call. */ +void lj_record_call(jit_State *J, BCReg func, ptrdiff_t nargs) +{ + rec_call_setup(J, func, nargs); + /* Bump frame. */ + J->framedepth++; + J->base += func+1; + J->baseslot += func+1; +} + +/* Record tail call. */ +void lj_record_tailcall(jit_State *J, BCReg func, ptrdiff_t nargs) +{ + rec_call_setup(J, func, nargs); + if (frame_isvarg(J->L->base - 1)) { + BCReg cbase = (BCReg)frame_delta(J->L->base - 1); + if (--J->framedepth < 0) + lj_trace_err(J, LJ_TRERR_NYIRETL); + J->baseslot -= (BCReg)cbase; + J->base -= cbase; + func += cbase; + } + /* Move func + args down. */ + memmove(&J->base[-1], &J->base[func], sizeof(TRef)*(J->maxslot+1)); + /* Note: the new TREF_FRAME is now at J->base[-1] (even for slot #0). */ + /* Tailcalls can form a loop, so count towards the loop unroll limit. */ + if (++J->tailcalled > J->loopunroll) + lj_trace_err(J, LJ_TRERR_LUNROLL); +} + +/* Check unroll limits for down-recursion. */ +static int check_downrec_unroll(jit_State *J, GCproto *pt) +{ + IRRef ptref; + for (ptref = J->chain[IR_KGC]; ptref; ptref = IR(ptref)->prev) + if (ir_kgc(IR(ptref)) == obj2gco(pt)) { + int count = 0; + IRRef ref; + for (ref = J->chain[IR_RETF]; ref; ref = IR(ref)->prev) + if (IR(ref)->op1 == ptref) + count++; + if (count) { + if (J->pc == J->startpc) { + if (count + J->tailcalled > J->param[JIT_P_recunroll]) + return 1; + } else { + lj_trace_err(J, LJ_TRERR_DOWNREC); + } + } + } + return 0; +} + +/* Record return. */ +void lj_record_ret(jit_State *J, BCReg rbase, ptrdiff_t gotresults) +{ + TValue *frame = J->L->base - 1; + ptrdiff_t i; + for (i = 0; i < gotresults; i++) + (void)getslot(J, rbase+i); /* Ensure all results have a reference. */ + while (frame_ispcall(frame)) { /* Immediately resolve pcall() returns. */ + BCReg cbase = (BCReg)frame_delta(frame); + if (--J->framedepth < 0) + lj_trace_err(J, LJ_TRERR_NYIRETL); + lua_assert(J->baseslot > 1); + gotresults++; + rbase += cbase; + J->baseslot -= (BCReg)cbase; + J->base -= cbase; + J->base[--rbase] = TREF_TRUE; /* Prepend true to results. */ + frame = frame_prevd(frame); + } + /* Return to lower frame via interpreter for unhandled cases. */ + if (J->framedepth == 0 && J->pt && bc_isret(bc_op(*J->pc)) && + (!frame_islua(frame) || + (J->parent == 0 && !bc_isret(bc_op(J->cur.startins))))) { + /* NYI: specialize to frame type and return directly, not via RET*. */ + for (i = -1; i < (ptrdiff_t)rbase; i++) + J->base[i] = 0; /* Purge dead slots. */ + J->maxslot = rbase + (BCReg)gotresults; + rec_stop(J, LJ_TRLINK_RETURN, 0); /* Return to interpreter. */ + return; + } + if (frame_isvarg(frame)) { + BCReg cbase = (BCReg)frame_delta(frame); + if (--J->framedepth < 0) /* NYI: return of vararg func to lower frame. */ + lj_trace_err(J, LJ_TRERR_NYIRETL); + lua_assert(J->baseslot > 1); + rbase += cbase; + J->baseslot -= (BCReg)cbase; + J->base -= cbase; + frame = frame_prevd(frame); + } + if (frame_islua(frame)) { /* Return to Lua frame. */ + BCIns callins = *(frame_pc(frame)-1); + ptrdiff_t nresults = bc_b(callins) ? (ptrdiff_t)bc_b(callins)-1 :gotresults; + BCReg cbase = bc_a(callins); + GCproto *pt = funcproto(frame_func(frame - (cbase+1))); + if (J->framedepth == 0 && J->pt && frame == J->L->base - 1) { + if (check_downrec_unroll(J, pt)) { + J->maxslot = (BCReg)(rbase + gotresults); + lj_snap_purge(J); + rec_stop(J, LJ_TRLINK_DOWNREC, J->cur.traceno); /* Down-recursion. */ + return; + } + lj_snap_add(J); + } + for (i = 0; i < nresults; i++) /* Adjust results. */ + J->base[i-1] = i < gotresults ? J->base[rbase+i] : TREF_NIL; + J->maxslot = cbase+(BCReg)nresults; + if (J->framedepth > 0) { /* Return to a frame that is part of the trace. */ + J->framedepth--; + lua_assert(J->baseslot > cbase+1); + J->baseslot -= cbase+1; + J->base -= cbase+1; + } else if (J->parent == 0 && !bc_isret(bc_op(J->cur.startins))) { + /* Return to lower frame would leave the loop in a root trace. */ + lj_trace_err(J, LJ_TRERR_LLEAVE); + } else { /* Return to lower frame. Guard for the target we return to. */ + TRef trpt = lj_ir_kgc(J, obj2gco(pt), IRT_PROTO); + TRef trpc = lj_ir_kptr(J, (void *)frame_pc(frame)); + emitir(IRTG(IR_RETF, IRT_P32), trpt, trpc); + J->retdepth++; + J->needsnap = 1; + lua_assert(J->baseslot == 1); + /* Shift result slots up and clear the slots of the new frame below. */ + memmove(J->base + cbase, J->base-1, sizeof(TRef)*nresults); + memset(J->base-1, 0, sizeof(TRef)*(cbase+1)); + } + } else if (frame_iscont(frame)) { /* Return to continuation frame. */ + ASMFunction cont = frame_contf(frame); + BCReg cbase = (BCReg)frame_delta(frame); + if ((J->framedepth -= 2) < 0) + lj_trace_err(J, LJ_TRERR_NYIRETL); + J->baseslot -= (BCReg)cbase; + J->base -= cbase; + J->maxslot = cbase-2; + if (cont == lj_cont_ra) { + /* Copy result to destination slot. */ + BCReg dst = bc_a(*(frame_contpc(frame)-1)); + J->base[dst] = gotresults ? J->base[cbase+rbase] : TREF_NIL; + if (dst >= J->maxslot) J->maxslot = dst+1; + } else if (cont == lj_cont_nop) { + /* Nothing to do here. */ + } else if (cont == lj_cont_cat) { + lua_assert(0); + } else { + /* Result type already specialized. */ + lua_assert(cont == lj_cont_condf || cont == lj_cont_condt); + } + } else { + lj_trace_err(J, LJ_TRERR_NYIRETL); /* NYI: handle return to C frame. */ + } + lua_assert(J->baseslot >= 1); +} + +/* -- Metamethod handling ------------------------------------------------- */ + +/* Prepare to record call to metamethod. */ +static BCReg rec_mm_prep(jit_State *J, ASMFunction cont) +{ + BCReg s, top = curr_proto(J->L)->framesize; + TRef trcont; + setcont(&J->L->base[top], cont); +#if LJ_64 + trcont = lj_ir_kptr(J, (void *)((int64_t)cont - (int64_t)lj_vm_asm_begin)); +#else + trcont = lj_ir_kptr(J, (void *)cont); +#endif + J->base[top] = trcont | TREF_CONT; + J->framedepth++; + for (s = J->maxslot; s < top; s++) + J->base[s] = 0; /* Clear frame gap to avoid resurrecting previous refs. */ + return top+1; +} + +/* Record metamethod lookup. */ +int lj_record_mm_lookup(jit_State *J, RecordIndex *ix, MMS mm) +{ + RecordIndex mix; + GCtab *mt; + if (tref_istab(ix->tab)) { + mt = tabref(tabV(&ix->tabv)->metatable); + mix.tab = emitir(IRT(IR_FLOAD, IRT_TAB), ix->tab, IRFL_TAB_META); + } else if (tref_isudata(ix->tab)) { + int udtype = udataV(&ix->tabv)->udtype; + mt = tabref(udataV(&ix->tabv)->metatable); + /* The metatables of special userdata objects are treated as immutable. */ + if (udtype != UDTYPE_USERDATA) { + cTValue *mo; + if (LJ_HASFFI && udtype == UDTYPE_FFI_CLIB) { + /* Specialize to the C library namespace object. */ + emitir(IRTG(IR_EQ, IRT_P32), ix->tab, lj_ir_kptr(J, udataV(&ix->tabv))); + } else { + /* Specialize to the type of userdata. */ + TRef tr = emitir(IRT(IR_FLOAD, IRT_U8), ix->tab, IRFL_UDATA_UDTYPE); + emitir(IRTGI(IR_EQ), tr, lj_ir_kint(J, udtype)); + } + immutable_mt: + mo = lj_tab_getstr(mt, mmname_str(J2G(J), mm)); + if (!mo || tvisnil(mo)) + return 0; /* No metamethod. */ + /* Treat metamethod or index table as immutable, too. */ + if (!(tvisfunc(mo) || tvistab(mo))) + lj_trace_err(J, LJ_TRERR_BADTYPE); + copyTV(J->L, &ix->mobjv, mo); + ix->mobj = lj_ir_kgc(J, gcV(mo), tvisfunc(mo) ? IRT_FUNC : IRT_TAB); + ix->mtv = mt; + ix->mt = TREF_NIL; /* Dummy value for comparison semantics. */ + return 1; /* Got metamethod or index table. */ + } + mix.tab = emitir(IRT(IR_FLOAD, IRT_TAB), ix->tab, IRFL_UDATA_META); + } else { + /* Specialize to base metatable. Must flush mcode in lua_setmetatable(). */ + mt = tabref(basemt_obj(J2G(J), &ix->tabv)); + if (mt == NULL) { + ix->mt = TREF_NIL; + return 0; /* No metamethod. */ + } + /* The cdata metatable is treated as immutable. */ + if (LJ_HASFFI && tref_iscdata(ix->tab)) goto immutable_mt; + ix->mt = mix.tab = lj_ir_ktab(J, mt); + goto nocheck; + } + ix->mt = mt ? mix.tab : TREF_NIL; + emitir(IRTG(mt ? IR_NE : IR_EQ, IRT_TAB), mix.tab, lj_ir_knull(J, IRT_TAB)); +nocheck: + if (mt) { + GCstr *mmstr = mmname_str(J2G(J), mm); + cTValue *mo = lj_tab_getstr(mt, mmstr); + if (mo && !tvisnil(mo)) + copyTV(J->L, &ix->mobjv, mo); + ix->mtv = mt; + settabV(J->L, &mix.tabv, mt); + setstrV(J->L, &mix.keyv, mmstr); + mix.key = lj_ir_kstr(J, mmstr); + mix.val = 0; + mix.idxchain = 0; + ix->mobj = lj_record_idx(J, &mix); + return !tref_isnil(ix->mobj); /* 1 if metamethod found, 0 if not. */ + } + return 0; /* No metamethod. */ +} + +/* Record call to arithmetic metamethod. */ +static TRef rec_mm_arith(jit_State *J, RecordIndex *ix, MMS mm) +{ + /* Set up metamethod call first to save ix->tab and ix->tabv. */ + BCReg func = rec_mm_prep(J, lj_cont_ra); + TRef *base = J->base + func; + TValue *basev = J->L->base + func; + base[1] = ix->tab; base[2] = ix->key; + copyTV(J->L, basev+1, &ix->tabv); + copyTV(J->L, basev+2, &ix->keyv); + if (!lj_record_mm_lookup(J, ix, mm)) { /* Lookup mm on 1st operand. */ + if (mm != MM_unm) { + ix->tab = ix->key; + copyTV(J->L, &ix->tabv, &ix->keyv); + if (lj_record_mm_lookup(J, ix, mm)) /* Lookup mm on 2nd operand. */ + goto ok; + } + lj_trace_err(J, LJ_TRERR_NOMM); + } +ok: + base[0] = ix->mobj; + copyTV(J->L, basev+0, &ix->mobjv); + lj_record_call(J, func, 2); + return 0; /* No result yet. */ +} + +/* Record call to __len metamethod. */ +static TRef rec_mm_len(jit_State *J, TRef tr, TValue *tv) +{ + RecordIndex ix; + ix.tab = tr; + copyTV(J->L, &ix.tabv, tv); + if (lj_record_mm_lookup(J, &ix, MM_len)) { + BCReg func = rec_mm_prep(J, lj_cont_ra); + TRef *base = J->base + func; + TValue *basev = J->L->base + func; + base[0] = ix.mobj; copyTV(J->L, basev+0, &ix.mobjv); + base[1] = tr; copyTV(J->L, basev+1, tv); +#ifdef LUAJIT_ENABLE_LUA52COMPAT + base[2] = tr; copyTV(J->L, basev+2, tv); +#else + base[2] = TREF_NIL; setnilV(basev+2); +#endif + lj_record_call(J, func, 2); + } else { +#ifdef LUAJIT_ENABLE_LUA52COMPAT + if (tref_istab(tr)) + return lj_ir_call(J, IRCALL_lj_tab_len, tr); +#endif + lj_trace_err(J, LJ_TRERR_NOMM); + } + return 0; /* No result yet. */ +} + +/* Call a comparison metamethod. */ +static void rec_mm_callcomp(jit_State *J, RecordIndex *ix, int op) +{ + BCReg func = rec_mm_prep(J, (op&1) ? lj_cont_condf : lj_cont_condt); + TRef *base = J->base + func; + TValue *tv = J->L->base + func; + base[0] = ix->mobj; base[1] = ix->val; base[2] = ix->key; + copyTV(J->L, tv+0, &ix->mobjv); + copyTV(J->L, tv+1, &ix->valv); + copyTV(J->L, tv+2, &ix->keyv); + lj_record_call(J, func, 2); +} + +/* Record call to equality comparison metamethod (for tab and udata only). */ +static void rec_mm_equal(jit_State *J, RecordIndex *ix, int op) +{ + ix->tab = ix->val; + copyTV(J->L, &ix->tabv, &ix->valv); + if (lj_record_mm_lookup(J, ix, MM_eq)) { /* Lookup mm on 1st operand. */ + cTValue *bv; + TRef mo1 = ix->mobj; + TValue mo1v; + copyTV(J->L, &mo1v, &ix->mobjv); + /* Avoid the 2nd lookup and the objcmp if the metatables are equal. */ + bv = &ix->keyv; + if (tvistab(bv) && tabref(tabV(bv)->metatable) == ix->mtv) { + TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_TAB_META); + emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt); + } else if (tvisudata(bv) && tabref(udataV(bv)->metatable) == ix->mtv) { + TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_UDATA_META); + emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt); + } else { /* Lookup metamethod on 2nd operand and compare both. */ + ix->tab = ix->key; + copyTV(J->L, &ix->tabv, bv); + if (!lj_record_mm_lookup(J, ix, MM_eq) || + lj_record_objcmp(J, mo1, ix->mobj, &mo1v, &ix->mobjv)) + return; + } + rec_mm_callcomp(J, ix, op); + } +} + +/* Record call to ordered comparison metamethods (for arbitrary objects). */ +static void rec_mm_comp(jit_State *J, RecordIndex *ix, int op) +{ + ix->tab = ix->val; + copyTV(J->L, &ix->tabv, &ix->valv); + while (1) { + MMS mm = (op & 2) ? MM_le : MM_lt; /* Try __le + __lt or only __lt. */ + if (lj_record_mm_lookup(J, ix, mm)) { /* Lookup mm on 1st operand. */ + cTValue *bv; + TRef mo1 = ix->mobj; + TValue mo1v; + copyTV(J->L, &mo1v, &ix->mobjv); + /* Avoid the 2nd lookup and the objcmp if the metatables are equal. */ + bv = &ix->keyv; + if (tvistab(bv) && tabref(tabV(bv)->metatable) == ix->mtv) { + TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_TAB_META); + emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt); + } else if (tvisudata(bv) && tabref(udataV(bv)->metatable) == ix->mtv) { + TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_UDATA_META); + emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt); + } else { /* Lookup metamethod on 2nd operand and compare both. */ + ix->tab = ix->key; + copyTV(J->L, &ix->tabv, bv); + if (!lj_record_mm_lookup(J, ix, mm) || + lj_record_objcmp(J, mo1, ix->mobj, &mo1v, &ix->mobjv)) + goto nomatch; + } + rec_mm_callcomp(J, ix, op); + return; + } + nomatch: + /* First lookup failed. Retry with __lt and swapped operands. */ + if (!(op & 2)) break; /* Already at __lt. Interpreter will throw. */ + ix->tab = ix->key; ix->key = ix->val; ix->val = ix->tab; + copyTV(J->L, &ix->tabv, &ix->keyv); + copyTV(J->L, &ix->keyv, &ix->valv); + copyTV(J->L, &ix->valv, &ix->tabv); + op ^= 3; + } +} + +#if LJ_HASFFI +/* Setup call to cdata comparison metamethod. */ +static void rec_mm_comp_cdata(jit_State *J, RecordIndex *ix, int op, MMS mm) +{ + lj_snap_add(J); + if (tref_iscdata(ix->val)) { + ix->tab = ix->val; + copyTV(J->L, &ix->tabv, &ix->valv); + } else { + lua_assert(tref_iscdata(ix->key)); + ix->tab = ix->key; + copyTV(J->L, &ix->tabv, &ix->keyv); + } + lj_record_mm_lookup(J, ix, mm); + rec_mm_callcomp(J, ix, op); +} +#endif + +/* -- Indexed access ------------------------------------------------------ */ + +/* Record bounds-check. */ +static void rec_idx_abc(jit_State *J, TRef asizeref, TRef ikey, uint32_t asize) +{ + /* Try to emit invariant bounds checks. */ + if ((J->flags & (JIT_F_OPT_LOOP|JIT_F_OPT_ABC)) == + (JIT_F_OPT_LOOP|JIT_F_OPT_ABC)) { + IRRef ref = tref_ref(ikey); + IRIns *ir = IR(ref); + int32_t ofs = 0; + IRRef ofsref = 0; + /* Handle constant offsets. */ + if (ir->o == IR_ADD && irref_isk(ir->op2)) { + ofsref = ir->op2; + ofs = IR(ofsref)->i; + ref = ir->op1; + ir = IR(ref); + } + /* Got scalar evolution analysis results for this reference? */ + if (ref == J->scev.idx) { + int32_t stop; + lua_assert(irt_isint(J->scev.t) && ir->o == IR_SLOAD); + stop = numberVint(&(J->L->base - J->baseslot)[ir->op1 + FORL_STOP]); + /* Runtime value for stop of loop is within bounds? */ + if ((int64_t)stop + ofs < (int64_t)asize) { + /* Emit invariant bounds check for stop. */ + emitir(IRTG(IR_ABC, IRT_P32), asizeref, ofs == 0 ? J->scev.stop : + emitir(IRTI(IR_ADD), J->scev.stop, ofsref)); + /* Emit invariant bounds check for start, if not const or negative. */ + if (!(J->scev.dir && J->scev.start && + (int64_t)IR(J->scev.start)->i + ofs >= 0)) + emitir(IRTG(IR_ABC, IRT_P32), asizeref, ikey); + return; + } + } + } + emitir(IRTGI(IR_ABC), asizeref, ikey); /* Emit regular bounds check. */ +} + +/* Record indexed key lookup. */ +static TRef rec_idx_key(jit_State *J, RecordIndex *ix) +{ + TRef key; + GCtab *t = tabV(&ix->tabv); + ix->oldv = lj_tab_get(J->L, t, &ix->keyv); /* Lookup previous value. */ + + /* Integer keys are looked up in the array part first. */ + key = ix->key; + if (tref_isnumber(key)) { + int32_t k = numberVint(&ix->keyv); + if (!tvisint(&ix->keyv) && numV(&ix->keyv) != (lua_Number)k) + k = LJ_MAX_ASIZE; + if ((MSize)k < LJ_MAX_ASIZE) { /* Potential array key? */ + TRef ikey = lj_opt_narrow_index(J, key); + TRef asizeref = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_ASIZE); + if ((MSize)k < t->asize) { /* Currently an array key? */ + TRef arrayref; + rec_idx_abc(J, asizeref, ikey, t->asize); + arrayref = emitir(IRT(IR_FLOAD, IRT_P32), ix->tab, IRFL_TAB_ARRAY); + return emitir(IRT(IR_AREF, IRT_P32), arrayref, ikey); + } else { /* Currently not in array (may be an array extension)? */ + emitir(IRTGI(IR_ULE), asizeref, ikey); /* Inv. bounds check. */ + if (k == 0 && tref_isk(key)) + key = lj_ir_knum_zero(J); /* Canonicalize 0 or +-0.0 to +0.0. */ + /* And continue with the hash lookup. */ + } + } else if (!tref_isk(key)) { + /* We can rule out const numbers which failed the integerness test + ** above. But all other numbers are potential array keys. + */ + if (t->asize == 0) { /* True sparse tables have an empty array part. */ + /* Guard that the array part stays empty. */ + TRef tmp = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_ASIZE); + emitir(IRTGI(IR_EQ), tmp, lj_ir_kint(J, 0)); + } else { + lj_trace_err(J, LJ_TRERR_NYITMIX); + } + } + } + + /* Otherwise the key is located in the hash part. */ + if (t->hmask == 0) { /* Shortcut for empty hash part. */ + /* Guard that the hash part stays empty. */ + TRef tmp = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_HMASK); + emitir(IRTGI(IR_EQ), tmp, lj_ir_kint(J, 0)); + return lj_ir_kkptr(J, niltvg(J2G(J))); + } + if (tref_isinteger(key)) /* Hash keys are based on numbers, not ints. */ + ix->key = key = emitir(IRTN(IR_CONV), key, IRCONV_NUM_INT); + if (tref_isk(key)) { + /* Optimize lookup of constant hash keys. */ + MSize hslot = (MSize)((char *)ix->oldv - (char *)&noderef(t->node)[0].val); + if (t->hmask > 0 && hslot <= t->hmask*(MSize)sizeof(Node) && + hslot <= 65535*(MSize)sizeof(Node)) { + TRef node, kslot; + TRef hm = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_HMASK); + emitir(IRTGI(IR_EQ), hm, lj_ir_kint(J, (int32_t)t->hmask)); + node = emitir(IRT(IR_FLOAD, IRT_P32), ix->tab, IRFL_TAB_NODE); + kslot = lj_ir_kslot(J, key, hslot / sizeof(Node)); + return emitir(IRTG(IR_HREFK, IRT_P32), node, kslot); + } + } + /* Fall back to a regular hash lookup. */ + return emitir(IRT(IR_HREF, IRT_P32), ix->tab, key); +} + +/* Determine whether a key is NOT one of the fast metamethod names. */ +static int nommstr(jit_State *J, TRef key) +{ + if (tref_isstr(key)) { + if (tref_isk(key)) { + GCstr *str = ir_kstr(IR(tref_ref(key))); + uint32_t mm; + for (mm = 0; mm <= MM_FAST; mm++) + if (mmname_str(J2G(J), mm) == str) + return 0; /* MUST be one the fast metamethod names. */ + } else { + return 0; /* Variable string key MAY be a metamethod name. */ + } + } + return 1; /* CANNOT be a metamethod name. */ +} + +/* Record indexed load/store. */ +TRef lj_record_idx(jit_State *J, RecordIndex *ix) +{ + TRef xref; + IROp xrefop, loadop; + cTValue *oldv; + + while (!tref_istab(ix->tab)) { /* Handle non-table lookup. */ + /* Never call raw lj_record_idx() on non-table. */ + lua_assert(ix->idxchain != 0); + if (!lj_record_mm_lookup(J, ix, ix->val ? MM_newindex : MM_index)) + lj_trace_err(J, LJ_TRERR_NOMM); + handlemm: + if (tref_isfunc(ix->mobj)) { /* Handle metamethod call. */ + BCReg func = rec_mm_prep(J, ix->val ? lj_cont_nop : lj_cont_ra); + TRef *base = J->base + func; + TValue *tv = J->L->base + func; + base[0] = ix->mobj; base[1] = ix->tab; base[2] = ix->key; + setfuncV(J->L, tv+0, funcV(&ix->mobjv)); + copyTV(J->L, tv+1, &ix->tabv); + copyTV(J->L, tv+2, &ix->keyv); + if (ix->val) { + base[3] = ix->val; + copyTV(J->L, tv+3, &ix->valv); + lj_record_call(J, func, 3); /* mobj(tab, key, val) */ + return 0; + } else { + lj_record_call(J, func, 2); /* res = mobj(tab, key) */ + return 0; /* No result yet. */ + } + } + /* Otherwise retry lookup with metaobject. */ + ix->tab = ix->mobj; + copyTV(J->L, &ix->tabv, &ix->mobjv); + if (--ix->idxchain == 0) + lj_trace_err(J, LJ_TRERR_IDXLOOP); + } + + /* First catch nil and NaN keys for tables. */ + if (tvisnil(&ix->keyv) || (tvisnum(&ix->keyv) && tvisnan(&ix->keyv))) { + if (ix->val) /* Better fail early. */ + lj_trace_err(J, LJ_TRERR_STORENN); + if (tref_isk(ix->key)) { + if (ix->idxchain && lj_record_mm_lookup(J, ix, MM_index)) + goto handlemm; + return TREF_NIL; + } + } + + /* Record the key lookup. */ + xref = rec_idx_key(J, ix); + xrefop = IR(tref_ref(xref))->o; + loadop = xrefop == IR_AREF ? IR_ALOAD : IR_HLOAD; + /* The lj_meta_tset() inconsistency is gone, but better play safe. */ + oldv = xrefop == IR_KKPTR ? (cTValue *)ir_kptr(IR(tref_ref(xref))) : ix->oldv; + + if (ix->val == 0) { /* Indexed load */ + IRType t = itype2irt(oldv); + TRef res; + if (oldv == niltvg(J2G(J))) { + emitir(IRTG(IR_EQ, IRT_P32), xref, lj_ir_kkptr(J, niltvg(J2G(J)))); + res = TREF_NIL; + } else { + res = emitir(IRTG(loadop, t), xref, 0); + } + if (t == IRT_NIL && ix->idxchain && lj_record_mm_lookup(J, ix, MM_index)) + goto handlemm; + if (irtype_ispri(t)) res = TREF_PRI(t); /* Canonicalize primitives. */ + return res; + } else { /* Indexed store. */ + GCtab *mt = tabref(tabV(&ix->tabv)->metatable); + int keybarrier = tref_isgcv(ix->key) && !tref_isnil(ix->val); + if (tvisnil(oldv)) { /* Previous value was nil? */ + /* Need to duplicate the hasmm check for the early guards. */ + int hasmm = 0; + if (ix->idxchain && mt) { + cTValue *mo = lj_tab_getstr(mt, mmname_str(J2G(J), MM_newindex)); + hasmm = mo && !tvisnil(mo); + } + if (hasmm) + emitir(IRTG(loadop, IRT_NIL), xref, 0); /* Guard for nil value. */ + else if (xrefop == IR_HREF) + emitir(IRTG(oldv == niltvg(J2G(J)) ? IR_EQ : IR_NE, IRT_P32), + xref, lj_ir_kkptr(J, niltvg(J2G(J)))); + if (ix->idxchain && lj_record_mm_lookup(J, ix, MM_newindex)) { + lua_assert(hasmm); + goto handlemm; + } + lua_assert(!hasmm); + if (oldv == niltvg(J2G(J))) { /* Need to insert a new key. */ + TRef key = ix->key; + if (tref_isinteger(key)) /* NEWREF needs a TValue as a key. */ + key = emitir(IRTN(IR_CONV), key, IRCONV_NUM_INT); + xref = emitir(IRT(IR_NEWREF, IRT_P32), ix->tab, key); + keybarrier = 0; /* NEWREF already takes care of the key barrier. */ + } + } else if (!lj_opt_fwd_wasnonnil(J, loadop, tref_ref(xref))) { + /* Cannot derive that the previous value was non-nil, must do checks. */ + if (xrefop == IR_HREF) /* Guard against store to niltv. */ + emitir(IRTG(IR_NE, IRT_P32), xref, lj_ir_kkptr(J, niltvg(J2G(J)))); + if (ix->idxchain) { /* Metamethod lookup required? */ + /* A check for NULL metatable is cheaper (hoistable) than a load. */ + if (!mt) { + TRef mtref = emitir(IRT(IR_FLOAD, IRT_TAB), ix->tab, IRFL_TAB_META); + emitir(IRTG(IR_EQ, IRT_TAB), mtref, lj_ir_knull(J, IRT_TAB)); + } else { + IRType t = itype2irt(oldv); + emitir(IRTG(loadop, t), xref, 0); /* Guard for non-nil value. */ + } + } + } else { + keybarrier = 0; /* Previous non-nil value kept the key alive. */ + } + /* Convert int to number before storing. */ + if (!LJ_DUALNUM && tref_isinteger(ix->val)) + ix->val = emitir(IRTN(IR_CONV), ix->val, IRCONV_NUM_INT); + emitir(IRT(loadop+IRDELTA_L2S, tref_type(ix->val)), xref, ix->val); + if (keybarrier || tref_isgcv(ix->val)) + emitir(IRT(IR_TBAR, IRT_NIL), ix->tab, 0); + /* Invalidate neg. metamethod cache for stores with certain string keys. */ + if (!nommstr(J, ix->key)) { + TRef fref = emitir(IRT(IR_FREF, IRT_P32), ix->tab, IRFL_TAB_NOMM); + emitir(IRT(IR_FSTORE, IRT_U8), fref, lj_ir_kint(J, 0)); + } + J->needsnap = 1; + return 0; + } +} + +/* -- Upvalue access ------------------------------------------------------ */ + +/* Record upvalue load/store. */ +static TRef rec_upvalue(jit_State *J, uint32_t uv, TRef val) +{ + GCupval *uvp = &gcref(J->fn->l.uvptr[uv])->uv; + TRef fn = getcurrf(J); + IRRef uref; + int needbarrier = 0; + /* Note: this effectively limits LJ_MAX_UPVAL to 127. */ + uv = (uv << 8) | (hashrot(uvp->dhash, uvp->dhash + HASH_BIAS) & 0xff); + if (!uvp->closed) { + /* In current stack? */ + if (uvval(uvp) >= tvref(J->L->stack) && + uvval(uvp) < tvref(J->L->maxstack)) { + int32_t slot = (int32_t)(uvval(uvp) - (J->L->base - J->baseslot)); + if (slot >= 0) { /* Aliases an SSA slot? */ + slot -= (int32_t)J->baseslot; /* Note: slot number may be negative! */ + /* NYI: add IR to guard that it's still aliasing the same slot. */ + if (val == 0) { + return getslot(J, slot); + } else { + J->base[slot] = val; + if (slot >= (int32_t)J->maxslot) J->maxslot = (BCReg)(slot+1); + return 0; + } + } + } + uref = tref_ref(emitir(IRTG(IR_UREFO, IRT_P32), fn, uv)); + } else { + needbarrier = 1; + uref = tref_ref(emitir(IRTG(IR_UREFC, IRT_P32), fn, uv)); + } + if (val == 0) { /* Upvalue load */ + IRType t = itype2irt(uvval(uvp)); + TRef res = emitir(IRTG(IR_ULOAD, t), uref, 0); + if (irtype_ispri(t)) res = TREF_PRI(t); /* Canonicalize primitive refs. */ + return res; + } else { /* Upvalue store. */ + /* Convert int to number before storing. */ + if (!LJ_DUALNUM && tref_isinteger(val)) + val = emitir(IRTN(IR_CONV), val, IRCONV_NUM_INT); + emitir(IRT(IR_USTORE, tref_type(val)), uref, val); + if (needbarrier && tref_isgcv(val)) + emitir(IRT(IR_OBAR, IRT_NIL), uref, val); + J->needsnap = 1; + return 0; + } +} + +/* -- Record calls to Lua functions --------------------------------------- */ + +/* Check unroll limits for calls. */ +static void check_call_unroll(jit_State *J, TraceNo lnk) +{ + cTValue *frame = J->L->base - 1; + void *pc = mref(frame_func(frame)->l.pc, void); + int32_t depth = J->framedepth; + int32_t count = 0; + if ((J->pt->flags & PROTO_VARARG)) depth--; /* Vararg frame still missing. */ + for (; depth > 0; depth--) { /* Count frames with same prototype. */ + frame = frame_prev(frame); + if (mref(frame_func(frame)->l.pc, void) == pc) + count++; + } + if (J->pc == J->startpc) { + if (count + J->tailcalled > J->param[JIT_P_recunroll]) { + J->pc++; + if (J->framedepth + J->retdepth == 0) + rec_stop(J, LJ_TRLINK_TAILREC, J->cur.traceno); /* Tail-recursion. */ + else + rec_stop(J, LJ_TRLINK_UPREC, J->cur.traceno); /* Up-recursion. */ + } + } else { + if (count > J->param[JIT_P_callunroll]) { + if (lnk) { /* Possible tail- or up-recursion. */ + lj_trace_flush(J, lnk); /* Flush trace that only returns. */ + /* Set a small, pseudo-random hotcount for a quick retry of JFUNC*. */ + hotcount_set(J2GG(J), J->pc+1, LJ_PRNG_BITS(J, 4)); + } + lj_trace_err(J, LJ_TRERR_CUNROLL); + } + } +} + +/* Record Lua function setup. */ +static void rec_func_setup(jit_State *J) +{ + GCproto *pt = J->pt; + BCReg s, numparams = pt->numparams; + if ((pt->flags & PROTO_NOJIT)) + lj_trace_err(J, LJ_TRERR_CJITOFF); + if (J->baseslot + pt->framesize >= LJ_MAX_JSLOTS) + lj_trace_err(J, LJ_TRERR_STACKOV); + /* Fill up missing parameters with nil. */ + for (s = J->maxslot; s < numparams; s++) + J->base[s] = TREF_NIL; + /* The remaining slots should never be read before they are written. */ + J->maxslot = numparams; +} + +/* Record Lua vararg function setup. */ +static void rec_func_vararg(jit_State *J) +{ + GCproto *pt = J->pt; + BCReg s, fixargs, vframe = J->maxslot+1; + lua_assert((pt->flags & PROTO_VARARG)); + if (J->baseslot + vframe + pt->framesize >= LJ_MAX_JSLOTS) + lj_trace_err(J, LJ_TRERR_STACKOV); + J->base[vframe-1] = J->base[-1]; /* Copy function up. */ + /* Copy fixarg slots up and set their original slots to nil. */ + fixargs = pt->numparams < J->maxslot ? pt->numparams : J->maxslot; + for (s = 0; s < fixargs; s++) { + J->base[vframe+s] = J->base[s]; + J->base[s] = TREF_NIL; + } + J->maxslot = fixargs; + J->framedepth++; + J->base += vframe; + J->baseslot += vframe; +} + +/* Record entry to a Lua function. */ +static void rec_func_lua(jit_State *J) +{ + rec_func_setup(J); + check_call_unroll(J, 0); +} + +/* Record entry to an already compiled function. */ +static void rec_func_jit(jit_State *J, TraceNo lnk) +{ + GCtrace *T; + rec_func_setup(J); + T = traceref(J, lnk); + if (T->linktype == LJ_TRLINK_RETURN) { /* Trace returns to interpreter? */ + check_call_unroll(J, lnk); + /* Temporarily unpatch JFUNC* to continue recording across function. */ + J->patchins = *J->pc; + J->patchpc = (BCIns *)J->pc; + *J->patchpc = T->startins; + return; + } + J->instunroll = 0; /* Cannot continue across a compiled function. */ + if (J->pc == J->startpc && J->framedepth + J->retdepth == 0) + rec_stop(J, LJ_TRLINK_TAILREC, J->cur.traceno); /* Extra tail-recursion. */ + else + rec_stop(J, LJ_TRLINK_ROOT, lnk); /* Link to the function. */ +} + +/* -- Vararg handling ----------------------------------------------------- */ + +/* Detect y = select(x, ...) idiom. */ +static int select_detect(jit_State *J) +{ + BCIns ins = J->pc[1]; + if (bc_op(ins) == BC_CALLM && bc_b(ins) == 2 && bc_c(ins) == 1) { + cTValue *func = &J->L->base[bc_a(ins)]; + if (tvisfunc(func) && funcV(func)->c.ffid == FF_select) + return 1; + } + return 0; +} + +/* Record vararg instruction. */ +static void rec_varg(jit_State *J, BCReg dst, ptrdiff_t nresults) +{ + int32_t numparams = J->pt->numparams; + ptrdiff_t nvararg = frame_delta(J->L->base-1) - numparams - 1; + lua_assert(frame_isvarg(J->L->base-1)); + if (J->framedepth > 0) { /* Simple case: varargs defined on-trace. */ + ptrdiff_t i; + if (nvararg < 0) nvararg = 0; + if (nresults == -1) { + nresults = nvararg; + J->maxslot = dst + (BCReg)nvararg; + } else if (dst + nresults > J->maxslot) { + J->maxslot = dst + (BCReg)nresults; + } + for (i = 0; i < nresults; i++) { + J->base[dst+i] = i < nvararg ? J->base[i - nvararg - 1] : TREF_NIL; + lua_assert(J->base[dst+i] != 0); + } + } else { /* Unknown number of varargs passed to trace. */ + TRef fr = emitir(IRTI(IR_SLOAD), 0, IRSLOAD_READONLY|IRSLOAD_FRAME); + int32_t frofs = 8*(1+numparams)+FRAME_VARG; + if (nresults >= 0) { /* Known fixed number of results. */ + ptrdiff_t i; + if (nvararg > 0) { + ptrdiff_t nload = nvararg >= nresults ? nresults : nvararg; + TRef vbase; + if (nvararg >= nresults) + emitir(IRTGI(IR_GE), fr, lj_ir_kint(J, frofs+8*(int32_t)nresults)); + else + emitir(IRTGI(IR_EQ), fr, lj_ir_kint(J, frame_ftsz(J->L->base-1))); + vbase = emitir(IRTI(IR_SUB), REF_BASE, fr); + vbase = emitir(IRT(IR_ADD, IRT_P32), vbase, lj_ir_kint(J, frofs-8)); + for (i = 0; i < nload; i++) { + IRType t = itype2irt(&J->L->base[i-1-nvararg]); + TRef aref = emitir(IRT(IR_AREF, IRT_P32), + vbase, lj_ir_kint(J, (int32_t)i)); + TRef tr = emitir(IRTG(IR_VLOAD, t), aref, 0); + if (irtype_ispri(t)) tr = TREF_PRI(t); /* Canonicalize primitives. */ + J->base[dst+i] = tr; + } + } else { + emitir(IRTGI(IR_LE), fr, lj_ir_kint(J, frofs)); + nvararg = 0; + } + for (i = nvararg; i < nresults; i++) + J->base[dst+i] = TREF_NIL; + if (dst + (BCReg)nresults > J->maxslot) + J->maxslot = dst + (BCReg)nresults; + } else if (select_detect(J)) { /* y = select(x, ...) */ + TRef tridx = J->base[dst-1]; + TRef tr = TREF_NIL; + ptrdiff_t idx = lj_ffrecord_select_mode(J, tridx, &J->L->base[dst-1]); + if (idx < 0) goto nyivarg; + if (idx != 0 && !tref_isinteger(tridx)) + tridx = emitir(IRTGI(IR_CONV), tridx, IRCONV_INT_NUM|IRCONV_INDEX); + if (idx != 0 && tref_isk(tridx)) { + emitir(IRTGI(idx <= nvararg ? IR_GE : IR_LT), + fr, lj_ir_kint(J, frofs+8*(int32_t)idx)); + frofs -= 8; /* Bias for 1-based index. */ + } else if (idx <= nvararg) { /* Compute size. */ + TRef tmp = emitir(IRTI(IR_ADD), fr, lj_ir_kint(J, -frofs)); + if (numparams) + emitir(IRTGI(IR_GE), tmp, lj_ir_kint(J, 0)); + tr = emitir(IRTI(IR_BSHR), tmp, lj_ir_kint(J, 3)); + if (idx != 0) { + tridx = emitir(IRTI(IR_ADD), tridx, lj_ir_kint(J, -1)); + rec_idx_abc(J, tr, tridx, (uint32_t)nvararg); + } + } else { + TRef tmp = lj_ir_kint(J, frofs); + if (idx != 0) { + TRef tmp2 = emitir(IRTI(IR_BSHL), tridx, lj_ir_kint(J, 3)); + tmp = emitir(IRTI(IR_ADD), tmp2, tmp); + } else { + tr = lj_ir_kint(J, 0); + } + emitir(IRTGI(IR_LT), fr, tmp); + } + if (idx != 0 && idx <= nvararg) { + IRType t; + TRef aref, vbase = emitir(IRTI(IR_SUB), REF_BASE, fr); + vbase = emitir(IRT(IR_ADD, IRT_P32), vbase, lj_ir_kint(J, frofs-8)); + t = itype2irt(&J->L->base[idx-2-nvararg]); + aref = emitir(IRT(IR_AREF, IRT_P32), vbase, tridx); + tr = emitir(IRTG(IR_VLOAD, t), aref, 0); + if (irtype_ispri(t)) tr = TREF_PRI(t); /* Canonicalize primitives. */ + } + J->base[dst-2] = tr; + J->maxslot = dst-1; + J->bcskip = 2; /* Skip CALLM + select. */ + } else { + nyivarg: + setintV(&J->errinfo, BC_VARG); + lj_trace_err_info(J, LJ_TRERR_NYIBC); + } + } +} + +/* -- Record allocations -------------------------------------------------- */ + +static TRef rec_tnew(jit_State *J, uint32_t ah) +{ + uint32_t asize = ah & 0x7ff; + uint32_t hbits = ah >> 11; + if (asize == 0x7ff) asize = 0x801; + return emitir(IRTG(IR_TNEW, IRT_TAB), asize, hbits); +} + +/* -- Record bytecode ops ------------------------------------------------- */ + +/* Prepare for comparison. */ +static void rec_comp_prep(jit_State *J) +{ + /* Prevent merging with snapshot #0 (GC exit) since we fixup the PC. */ + if (J->cur.nsnap == 1 && J->cur.snap[0].ref == J->cur.nins) + emitir_raw(IRT(IR_NOP, IRT_NIL), 0, 0); + lj_snap_add(J); +} + +/* Fixup comparison. */ +static void rec_comp_fixup(jit_State *J, const BCIns *pc, int cond) +{ + BCIns jmpins = pc[1]; + const BCIns *npc = pc + 2 + (cond ? bc_j(jmpins) : 0); + SnapShot *snap = &J->cur.snap[J->cur.nsnap-1]; + /* Set PC to opposite target to avoid re-recording the comp. in side trace. */ + J->cur.snapmap[snap->mapofs + snap->nent] = SNAP_MKPC(npc); + J->needsnap = 1; + if (bc_a(jmpins) < J->maxslot) J->maxslot = bc_a(jmpins); + lj_snap_shrink(J); /* Shrink last snapshot if possible. */ +} + +/* Record the next bytecode instruction (_before_ it's executed). */ +void lj_record_ins(jit_State *J) +{ + cTValue *lbase; + RecordIndex ix; + const BCIns *pc; + BCIns ins; + BCOp op; + TRef ra, rb, rc; + + /* Perform post-processing action before recording the next instruction. */ + if (LJ_UNLIKELY(J->postproc != LJ_POST_NONE)) { + switch (J->postproc) { + case LJ_POST_FIXCOMP: /* Fixup comparison. */ + pc = frame_pc(&J2G(J)->tmptv); + rec_comp_fixup(J, pc, (!tvistruecond(&J2G(J)->tmptv2) ^ (bc_op(*pc)&1))); + /* fallthrough */ + case LJ_POST_FIXGUARD: /* Fixup and emit pending guard. */ + case LJ_POST_FIXGUARDSNAP: /* Fixup and emit pending guard and snapshot. */ + if (!tvistruecond(&J2G(J)->tmptv2)) { + J->fold.ins.o ^= 1; /* Flip guard to opposite. */ + if (J->postproc == LJ_POST_FIXGUARDSNAP) { + SnapShot *snap = &J->cur.snap[J->cur.nsnap-1]; + J->cur.snapmap[snap->mapofs+snap->nent-1]--; /* False -> true. */ + } + } + lj_opt_fold(J); /* Emit pending guard. */ + /* fallthrough */ + case LJ_POST_FIXBOOL: + if (!tvistruecond(&J2G(J)->tmptv2)) { + BCReg s; + for (s = 0; s < J->maxslot; s++) /* Fixup stack slot (if any). */ + if (J->base[s] == TREF_TRUE && tvisfalse(&J->L->base[s])) { + J->base[s] = TREF_FALSE; + break; + } + } + break; + case LJ_POST_FFRETRY: /* Suppress recording of retried fast function. */ + if (bc_op(*J->pc) >= BC__MAX) + return; + break; + default: lua_assert(0); break; + } + J->postproc = LJ_POST_NONE; + } + + /* Need snapshot before recording next bytecode (e.g. after a store). */ + if (J->needsnap) { + J->needsnap = 0; + lj_snap_purge(J); + lj_snap_add(J); + J->mergesnap = 1; + } + + /* Skip some bytecodes. */ + if (LJ_UNLIKELY(J->bcskip > 0)) { + J->bcskip--; + return; + } + + /* Record only closed loops for root traces. */ + pc = J->pc; + if (J->framedepth == 0 && + (MSize)((char *)pc - (char *)J->bc_min) >= J->bc_extent) + lj_trace_err(J, LJ_TRERR_LLEAVE); + +#ifdef LUA_USE_ASSERT + rec_check_slots(J); + rec_check_ir(J); +#endif + + /* Keep a copy of the runtime values of var/num/str operands. */ +#define rav (&ix.valv) +#define rbv (&ix.tabv) +#define rcv (&ix.keyv) + + lbase = J->L->base; + ins = *pc; + op = bc_op(ins); + ra = bc_a(ins); + ix.val = 0; + switch (bcmode_a(op)) { + case BCMvar: + copyTV(J->L, rav, &lbase[ra]); ix.val = ra = getslot(J, ra); break; + default: break; /* Handled later. */ + } + rb = bc_b(ins); + rc = bc_c(ins); + switch (bcmode_b(op)) { + case BCMnone: rb = 0; rc = bc_d(ins); break; /* Upgrade rc to 'rd'. */ + case BCMvar: + copyTV(J->L, rbv, &lbase[rb]); ix.tab = rb = getslot(J, rb); break; + default: break; /* Handled later. */ + } + switch (bcmode_c(op)) { + case BCMvar: + copyTV(J->L, rcv, &lbase[rc]); ix.key = rc = getslot(J, rc); break; + case BCMpri: setitype(rcv, ~rc); ix.key = rc = TREF_PRI(IRT_NIL+rc); break; + case BCMnum: { cTValue *tv = proto_knumtv(J->pt, rc); + copyTV(J->L, rcv, tv); ix.key = rc = tvisint(tv) ? lj_ir_kint(J, intV(tv)) : + lj_ir_knumint(J, numV(tv)); } break; + case BCMstr: { GCstr *s = gco2str(proto_kgc(J->pt, ~(ptrdiff_t)rc)); + setstrV(J->L, rcv, s); ix.key = rc = lj_ir_kstr(J, s); } break; + default: break; /* Handled later. */ + } + + switch (op) { + + /* -- Comparison ops ---------------------------------------------------- */ + + case BC_ISLT: case BC_ISGE: case BC_ISLE: case BC_ISGT: +#if LJ_HASFFI + if (tref_iscdata(ra) || tref_iscdata(rc)) { + rec_mm_comp_cdata(J, &ix, op, ((int)op & 2) ? MM_le : MM_lt); + break; + } +#endif + /* Emit nothing for two numeric or string consts. */ + if (!(tref_isk2(ra,rc) && tref_isnumber_str(ra) && tref_isnumber_str(rc))) { + IRType ta = tref_isinteger(ra) ? IRT_INT : tref_type(ra); + IRType tc = tref_isinteger(rc) ? IRT_INT : tref_type(rc); + int irop; + if (ta != tc) { + /* Widen mixed number/int comparisons to number/number comparison. */ + if (ta == IRT_INT && tc == IRT_NUM) { + ra = emitir(IRTN(IR_CONV), ra, IRCONV_NUM_INT); + ta = IRT_NUM; + } else if (ta == IRT_NUM && tc == IRT_INT) { + rc = emitir(IRTN(IR_CONV), rc, IRCONV_NUM_INT); + } else if (!((ta == IRT_FALSE || ta == IRT_TRUE) && + (tc == IRT_FALSE || tc == IRT_TRUE))) { + break; /* Interpreter will throw for two different types. */ + } + } + rec_comp_prep(J); + irop = (int)op - (int)BC_ISLT + (int)IR_LT; + if (ta == IRT_NUM) { + if ((irop & 1)) irop ^= 4; /* ISGE/ISGT are unordered. */ + if (!lj_ir_numcmp(numberVnum(rav), numberVnum(rcv), (IROp)irop)) + irop ^= 5; + } else if (ta == IRT_INT) { + if (!lj_ir_numcmp(numberVnum(rav), numberVnum(rcv), (IROp)irop)) + irop ^= 1; + } else if (ta == IRT_STR) { + if (!lj_ir_strcmp(strV(rav), strV(rcv), (IROp)irop)) irop ^= 1; + ra = lj_ir_call(J, IRCALL_lj_str_cmp, ra, rc); + rc = lj_ir_kint(J, 0); + ta = IRT_INT; + } else { + rec_mm_comp(J, &ix, (int)op); + break; + } + emitir(IRTG(irop, ta), ra, rc); + rec_comp_fixup(J, J->pc, ((int)op ^ irop) & 1); + } + break; + + case BC_ISEQV: case BC_ISNEV: + case BC_ISEQS: case BC_ISNES: + case BC_ISEQN: case BC_ISNEN: + case BC_ISEQP: case BC_ISNEP: +#if LJ_HASFFI + if (tref_iscdata(ra) || tref_iscdata(rc)) { + rec_mm_comp_cdata(J, &ix, op, MM_eq); + break; + } +#endif + /* Emit nothing for two non-table, non-udata consts. */ + if (!(tref_isk2(ra, rc) && !(tref_istab(ra) || tref_isudata(ra)))) { + int diff; + rec_comp_prep(J); + diff = lj_record_objcmp(J, ra, rc, rav, rcv); + if (diff == 1 && (tref_istab(ra) || tref_isudata(ra))) { + /* Only check __eq if different, but the same type (table or udata). */ + rec_mm_equal(J, &ix, (int)op); + break; + } + rec_comp_fixup(J, J->pc, ((int)op & 1) == !diff); + } + break; + + /* -- Unary test and copy ops ------------------------------------------- */ + + case BC_ISTC: case BC_ISFC: + if ((op & 1) == tref_istruecond(rc)) + rc = 0; /* Don't store if condition is not true. */ + /* fallthrough */ + case BC_IST: case BC_ISF: /* Type specialization suffices. */ + if (bc_a(pc[1]) < J->maxslot) + J->maxslot = bc_a(pc[1]); /* Shrink used slots. */ + break; + + /* -- Unary ops --------------------------------------------------------- */ + + case BC_NOT: + /* Type specialization already forces const result. */ + rc = tref_istruecond(rc) ? TREF_FALSE : TREF_TRUE; + break; + + case BC_LEN: + if (tref_isstr(rc)) + rc = emitir(IRTI(IR_FLOAD), rc, IRFL_STR_LEN); +#ifndef LUAJIT_ENABLE_LUA52COMPAT + else if (tref_istab(rc)) + rc = lj_ir_call(J, IRCALL_lj_tab_len, rc); +#endif + else + rc = rec_mm_len(J, rc, rcv); + break; + + /* -- Arithmetic ops ---------------------------------------------------- */ + + case BC_UNM: + if (tref_isnumber_str(rc)) { + rc = lj_opt_narrow_unm(J, rc, rcv); + } else { + ix.tab = rc; + copyTV(J->L, &ix.tabv, rcv); + rc = rec_mm_arith(J, &ix, MM_unm); + } + break; + + case BC_ADDNV: case BC_SUBNV: case BC_MULNV: case BC_DIVNV: case BC_MODNV: + /* Swap rb/rc and rbv/rcv. rav is temp. */ + ix.tab = rc; ix.key = rc = rb; rb = ix.tab; + copyTV(J->L, rav, rbv); + copyTV(J->L, rbv, rcv); + copyTV(J->L, rcv, rav); + if (op == BC_MODNV) + goto recmod; + /* fallthrough */ + case BC_ADDVN: case BC_SUBVN: case BC_MULVN: case BC_DIVVN: + case BC_ADDVV: case BC_SUBVV: case BC_MULVV: case BC_DIVVV: { + MMS mm = bcmode_mm(op); + if (tref_isnumber_str(rb) && tref_isnumber_str(rc)) + rc = lj_opt_narrow_arith(J, rb, rc, rbv, rcv, + (int)mm - (int)MM_add + (int)IR_ADD); + else + rc = rec_mm_arith(J, &ix, mm); + break; + } + + case BC_MODVN: case BC_MODVV: + recmod: + if (tref_isnumber_str(rb) && tref_isnumber_str(rc)) + rc = lj_opt_narrow_mod(J, rb, rc, rcv); + else + rc = rec_mm_arith(J, &ix, MM_mod); + break; + + case BC_POW: + if (tref_isnumber_str(rb) && tref_isnumber_str(rc)) + rc = lj_opt_narrow_pow(J, lj_ir_tonum(J, rb), rc, rcv); + else + rc = rec_mm_arith(J, &ix, MM_pow); + break; + + /* -- Constant and move ops --------------------------------------------- */ + + case BC_MOV: + /* Clear gap of method call to avoid resurrecting previous refs. */ + if (ra > J->maxslot) J->base[ra-1] = 0; + break; + case BC_KSTR: case BC_KNUM: case BC_KPRI: + break; + case BC_KSHORT: + rc = lj_ir_kint(J, (int32_t)(int16_t)rc); + break; + case BC_KNIL: + while (ra <= rc) + J->base[ra++] = TREF_NIL; + if (rc >= J->maxslot) J->maxslot = rc+1; + break; +#if LJ_HASFFI + case BC_KCDATA: + rc = lj_ir_kgc(J, proto_kgc(J->pt, ~(ptrdiff_t)rc), IRT_CDATA); + break; +#endif + + /* -- Upvalue and function ops ------------------------------------------ */ + + case BC_UGET: + rc = rec_upvalue(J, rc, 0); + break; + case BC_USETV: case BC_USETS: case BC_USETN: case BC_USETP: + rec_upvalue(J, ra, rc); + break; + + /* -- Table ops --------------------------------------------------------- */ + + case BC_GGET: case BC_GSET: + settabV(J->L, &ix.tabv, tabref(J->fn->l.env)); + ix.tab = emitir(IRT(IR_FLOAD, IRT_TAB), getcurrf(J), IRFL_FUNC_ENV); + ix.idxchain = LJ_MAX_IDXCHAIN; + rc = lj_record_idx(J, &ix); + break; + + case BC_TGETB: case BC_TSETB: + setintV(&ix.keyv, (int32_t)rc); + ix.key = lj_ir_kint(J, (int32_t)rc); + /* fallthrough */ + case BC_TGETV: case BC_TGETS: case BC_TSETV: case BC_TSETS: + ix.idxchain = LJ_MAX_IDXCHAIN; + rc = lj_record_idx(J, &ix); + break; + + case BC_TNEW: + rc = rec_tnew(J, rc); + break; + case BC_TDUP: + rc = emitir(IRTG(IR_TDUP, IRT_TAB), + lj_ir_ktab(J, gco2tab(proto_kgc(J->pt, ~(ptrdiff_t)rc))), 0); + break; + + /* -- Calls and vararg handling ----------------------------------------- */ + + case BC_ITERC: + J->base[ra] = getslot(J, ra-3); + J->base[ra+1] = getslot(J, ra-2); + J->base[ra+2] = getslot(J, ra-1); + { /* Do the actual copy now because lj_record_call needs the values. */ + TValue *b = &J->L->base[ra]; + copyTV(J->L, b, b-3); + copyTV(J->L, b+1, b-2); + copyTV(J->L, b+2, b-1); + } + lj_record_call(J, ra, (ptrdiff_t)rc-1); + break; + + /* L->top is set to L->base+ra+rc+NARGS-1+1. See lj_dispatch_ins(). */ + case BC_CALLM: + rc = (BCReg)(J->L->top - J->L->base) - ra; + /* fallthrough */ + case BC_CALL: + lj_record_call(J, ra, (ptrdiff_t)rc-1); + break; + + case BC_CALLMT: + rc = (BCReg)(J->L->top - J->L->base) - ra; + /* fallthrough */ + case BC_CALLT: + lj_record_tailcall(J, ra, (ptrdiff_t)rc-1); + break; + + case BC_VARG: + rec_varg(J, ra, (ptrdiff_t)rb-1); + break; + + /* -- Returns ----------------------------------------------------------- */ + + case BC_RETM: + /* L->top is set to L->base+ra+rc+NRESULTS-1, see lj_dispatch_ins(). */ + rc = (BCReg)(J->L->top - J->L->base) - ra + 1; + /* fallthrough */ + case BC_RET: case BC_RET0: case BC_RET1: + lj_record_ret(J, ra, (ptrdiff_t)rc-1); + break; + + /* -- Loops and branches ------------------------------------------------ */ + + case BC_FORI: + if (rec_for(J, pc, 0) != LOOPEV_LEAVE) + J->loopref = J->cur.nins; + break; + case BC_JFORI: + lua_assert(bc_op(pc[(ptrdiff_t)rc-BCBIAS_J]) == BC_JFORL); + if (rec_for(J, pc, 0) != LOOPEV_LEAVE) /* Link to existing loop. */ + rec_stop(J, LJ_TRLINK_ROOT, bc_d(pc[(ptrdiff_t)rc-BCBIAS_J])); + /* Continue tracing if the loop is not entered. */ + break; + + case BC_FORL: + rec_loop_interp(J, pc, rec_for(J, pc+((ptrdiff_t)rc-BCBIAS_J), 1)); + break; + case BC_ITERL: + rec_loop_interp(J, pc, rec_iterl(J, *pc)); + break; + case BC_LOOP: + rec_loop_interp(J, pc, rec_loop(J, ra)); + break; + + case BC_JFORL: + rec_loop_jit(J, rc, rec_for(J, pc+bc_j(traceref(J, rc)->startins), 1)); + break; + case BC_JITERL: + rec_loop_jit(J, rc, rec_iterl(J, traceref(J, rc)->startins)); + break; + case BC_JLOOP: + rec_loop_jit(J, rc, rec_loop(J, ra)); + break; + + case BC_IFORL: + case BC_IITERL: + case BC_ILOOP: + case BC_IFUNCF: + case BC_IFUNCV: + lj_trace_err(J, LJ_TRERR_BLACKL); + break; + + case BC_JMP: + if (ra < J->maxslot) + J->maxslot = ra; /* Shrink used slots. */ + break; + + /* -- Function headers -------------------------------------------------- */ + + case BC_FUNCF: + rec_func_lua(J); + break; + case BC_JFUNCF: + rec_func_jit(J, rc); + break; + + case BC_FUNCV: + rec_func_vararg(J); + rec_func_lua(J); + break; + case BC_JFUNCV: + lua_assert(0); /* Cannot happen. No hotcall counting for varag funcs. */ + break; + + case BC_FUNCC: + case BC_FUNCCW: + lj_ffrecord_func(J); + break; + + default: + if (op >= BC__MAX) { + lj_ffrecord_func(J); + break; + } + /* fallthrough */ + case BC_ITERN: + case BC_ISNEXT: + case BC_CAT: + case BC_UCLO: + case BC_FNEW: + case BC_TSETM: + setintV(&J->errinfo, (int32_t)op); + lj_trace_err_info(J, LJ_TRERR_NYIBC); + break; + } + + /* rc == 0 if we have no result yet, e.g. pending __index metamethod call. */ + if (bcmode_a(op) == BCMdst && rc) { + J->base[ra] = rc; + if (ra >= J->maxslot) J->maxslot = ra+1; + } + +#undef rav +#undef rbv +#undef rcv + + /* Limit the number of recorded IR instructions. */ + if (J->cur.nins > REF_FIRST+(IRRef)J->param[JIT_P_maxrecord]) + lj_trace_err(J, LJ_TRERR_TRACEOV); +} + +/* -- Recording setup ----------------------------------------------------- */ + +/* Setup recording for a root trace started by a hot loop. */ +static const BCIns *rec_setup_root(jit_State *J) +{ + /* Determine the next PC and the bytecode range for the loop. */ + const BCIns *pcj, *pc = J->pc; + BCIns ins = *pc; + BCReg ra = bc_a(ins); + switch (bc_op(ins)) { + case BC_FORL: + J->bc_extent = (MSize)(-bc_j(ins))*sizeof(BCIns); + pc += 1+bc_j(ins); + J->bc_min = pc; + break; + case BC_ITERL: + lua_assert(bc_op(pc[-1]) == BC_ITERC); + J->maxslot = ra + bc_b(pc[-1]) - 1; + J->bc_extent = (MSize)(-bc_j(ins))*sizeof(BCIns); + pc += 1+bc_j(ins); + lua_assert(bc_op(pc[-1]) == BC_JMP); + J->bc_min = pc; + break; + case BC_LOOP: + /* Only check BC range for real loops, but not for "repeat until true". */ + pcj = pc + bc_j(ins); + ins = *pcj; + if (bc_op(ins) == BC_JMP && bc_j(ins) < 0) { + J->bc_min = pcj+1 + bc_j(ins); + J->bc_extent = (MSize)(-bc_j(ins))*sizeof(BCIns); + } + J->maxslot = ra; + pc++; + break; + case BC_RET: + case BC_RET0: + case BC_RET1: + /* No bytecode range check for down-recursive root traces. */ + J->maxslot = ra + bc_d(ins); + break; + case BC_FUNCF: + /* No bytecode range check for root traces started by a hot call. */ + J->maxslot = J->pt->numparams; + pc++; + break; + default: + lua_assert(0); + break; + } + return pc; +} + +/* Setup recording for a side trace. */ +static void rec_setup_side(jit_State *J, GCtrace *T) +{ + SnapShot *snap = &T->snap[J->exitno]; + SnapEntry *map = &T->snapmap[snap->mapofs]; + MSize n, nent = snap->nent; + BloomFilter seen = 0; + J->framedepth = 0; + /* Emit IR for slots inherited from parent snapshot. */ + for (n = 0; n < nent; n++) { + SnapEntry sn = map[n]; + IRRef ref = snap_ref(sn); + BCReg s = snap_slot(sn); + IRIns *ir = &T->ir[ref]; + IRType t = irt_type(ir->t); + TRef tr; + /* The bloom filter avoids O(nent^2) overhead for de-duping slots. */ + if (bloomtest(seen, ref)) { + MSize j; + for (j = 0; j < n; j++) + if (snap_ref(map[j]) == ref) { + tr = J->slot[snap_slot(map[j])]; + goto setslot; + } + } + bloomset(seen, ref); + switch ((IROp)ir->o) { + /* Only have to deal with constants that can occur in stack slots. */ + case IR_KPRI: tr = TREF_PRI(t); break; + case IR_KINT: tr = lj_ir_kint(J, ir->i); break; + case IR_KGC: tr = lj_ir_kgc(J, ir_kgc(ir), irt_t(ir->t)); break; + case IR_KNUM: tr = lj_ir_k64(J, IR_KNUM, ir_knum(ir)); break; + case IR_KINT64: tr = lj_ir_k64(J, IR_KINT64, ir_kint64(ir)); break; + case IR_KPTR: tr = lj_ir_kptr(J, ir_kptr(ir)); break; /* Continuation. */ + /* Inherited SLOADs don't need a guard or type check. */ + case IR_SLOAD: + if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) t = IRT_NUM; + tr = emitir_raw(IRT(IR_SLOAD, t), s, + (ir->op2&IRSLOAD_READONLY) | IRSLOAD_INHERIT|IRSLOAD_PARENT); + break; + /* Parent refs are already typed and don't need a guard. */ + default: + if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) t = IRT_NUM; + tr = emitir_raw(IRT(IR_SLOAD, t), s, IRSLOAD_INHERIT|IRSLOAD_PARENT); + break; + } + setslot: + J->slot[s] = tr | (sn&(SNAP_CONT|SNAP_FRAME)); /* Same as TREF_* flags. */ + J->framedepth += ((sn & (SNAP_CONT|SNAP_FRAME)) && s); + if ((sn & SNAP_FRAME)) + J->baseslot = s+1; + } + J->base = J->slot + J->baseslot; + J->maxslot = snap->nslots - J->baseslot; + lj_snap_add(J); +} + +/* Setup for recording a new trace. */ +void lj_record_setup(jit_State *J) +{ + uint32_t i; + + /* Initialize state related to current trace. */ + memset(J->slot, 0, sizeof(J->slot)); + memset(J->chain, 0, sizeof(J->chain)); + memset(J->bpropcache, 0, sizeof(J->bpropcache)); + J->scev.idx = REF_NIL; + + J->baseslot = 1; /* Invoking function is at base[-1]. */ + J->base = J->slot + J->baseslot; + J->maxslot = 0; + J->framedepth = 0; + J->retdepth = 0; + + J->instunroll = J->param[JIT_P_instunroll]; + J->loopunroll = J->param[JIT_P_loopunroll]; + J->tailcalled = 0; + J->loopref = 0; + + J->bc_min = NULL; /* Means no limit. */ + J->bc_extent = ~(MSize)0; + + /* Emit instructions for fixed references. Also triggers initial IR alloc. */ + emitir_raw(IRT(IR_BASE, IRT_P32), J->parent, J->exitno); + for (i = 0; i <= 2; i++) { + IRIns *ir = IR(REF_NIL-i); + ir->i = 0; + ir->t.irt = (uint8_t)(IRT_NIL+i); + ir->o = IR_KPRI; + ir->prev = 0; + } + J->cur.nk = REF_TRUE; + + J->startpc = J->pc; + setmref(J->cur.startpc, J->pc); + if (J->parent) { /* Side trace. */ + GCtrace *T = traceref(J, J->parent); + TraceNo root = T->root ? T->root : J->parent; + J->cur.root = (uint16_t)root; + J->cur.startins = BCINS_AD(BC_JMP, 0, 0); + /* Check whether we could at least potentially form an extra loop. */ + if (J->exitno == 0 && T->snap[0].nent == 0) { + /* We can narrow a FORL for some side traces, too. */ + if (J->pc > proto_bc(J->pt) && bc_op(J->pc[-1]) == BC_JFORI && + bc_d(J->pc[bc_j(J->pc[-1])-1]) == root) { + lj_snap_add(J); + rec_for_loop(J, J->pc-1, &J->scev, 1); + goto sidecheck; + } + } else { + J->startpc = NULL; /* Prevent forming an extra loop. */ + } + rec_setup_side(J, T); + sidecheck: + if (traceref(J, J->cur.root)->nchild >= J->param[JIT_P_maxside] || + T->snap[J->exitno].count >= J->param[JIT_P_hotexit] + + J->param[JIT_P_tryside]) { + rec_stop(J, LJ_TRLINK_INTERP, 0); + } + } else { /* Root trace. */ + J->cur.root = 0; + J->cur.startins = *J->pc; + J->pc = rec_setup_root(J); + /* Note: the loop instruction itself is recorded at the end and not + ** at the start! So snapshot #0 needs to point to the *next* instruction. + */ + lj_snap_add(J); + if (bc_op(J->cur.startins) == BC_FORL) + rec_for_loop(J, J->pc-1, &J->scev, 1); + if (1 + J->pt->framesize >= LJ_MAX_JSLOTS) + lj_trace_err(J, LJ_TRERR_STACKOV); + } +#ifdef LUAJIT_ENABLE_CHECKHOOK + /* Regularly check for instruction/line hooks from compiled code and + ** exit to the interpreter if the hooks are set. + ** + ** This is a compile-time option and disabled by default, since the + ** hook checks may be quite expensive in tight loops. + ** + ** Note this is only useful if hooks are *not* set most of the time. + ** Use this only if you want to *asynchronously* interrupt the execution. + ** + ** You can set the instruction hook via lua_sethook() with a count of 1 + ** from a signal handler or another native thread. Please have a look + ** at the first few functions in luajit.c for an example (Ctrl-C handler). + */ + { + TRef tr = emitir(IRT(IR_XLOAD, IRT_U8), + lj_ir_kptr(J, &J2G(J)->hookmask), IRXLOAD_VOLATILE); + tr = emitir(IRTI(IR_BAND), tr, lj_ir_kint(J, (LUA_MASKLINE|LUA_MASKCOUNT))); + emitir(IRTGI(IR_EQ), tr, lj_ir_kint(J, 0)); + } +#endif +} + +#undef IR +#undef emitir_raw +#undef emitir + +#endif -- cgit v1.1