diff options
Diffstat (limited to '')
-rw-r--r-- | libraries/luajit-2.0/src/lj_asm_arm.h | 1785 |
1 files changed, 0 insertions, 1785 deletions
diff --git a/libraries/luajit-2.0/src/lj_asm_arm.h b/libraries/luajit-2.0/src/lj_asm_arm.h deleted file mode 100644 index 087fc0f..0000000 --- a/libraries/luajit-2.0/src/lj_asm_arm.h +++ /dev/null | |||
@@ -1,1785 +0,0 @@ | |||
1 | /* | ||
2 | ** ARM IR assembler (SSA IR -> machine code). | ||
3 | ** Copyright (C) 2005-2011 Mike Pall. See Copyright Notice in luajit.h | ||
4 | */ | ||
5 | |||
6 | /* -- Register allocator extensions --------------------------------------- */ | ||
7 | |||
8 | /* Allocate a register with a hint. */ | ||
9 | static Reg ra_hintalloc(ASMState *as, IRRef ref, Reg hint, RegSet allow) | ||
10 | { | ||
11 | Reg r = IR(ref)->r; | ||
12 | if (ra_noreg(r)) { | ||
13 | if (!ra_hashint(r) && !iscrossref(as, ref)) | ||
14 | ra_sethint(IR(ref)->r, hint); /* Propagate register hint. */ | ||
15 | r = ra_allocref(as, ref, allow); | ||
16 | } | ||
17 | ra_noweak(as, r); | ||
18 | return r; | ||
19 | } | ||
20 | |||
21 | /* Allocate a scratch register pair. */ | ||
22 | static Reg ra_scratchpair(ASMState *as, RegSet allow) | ||
23 | { | ||
24 | RegSet pick1 = as->freeset & allow; | ||
25 | RegSet pick2 = pick1 & (pick1 >> 1) & RSET_GPREVEN; | ||
26 | Reg r; | ||
27 | if (pick2) { | ||
28 | r = rset_picktop(pick2); | ||
29 | } else { | ||
30 | RegSet pick = pick1 & (allow >> 1) & RSET_GPREVEN; | ||
31 | if (pick) { | ||
32 | r = rset_picktop(pick); | ||
33 | ra_restore(as, regcost_ref(as->cost[r+1])); | ||
34 | } else { | ||
35 | pick = pick1 & (allow << 1) & RSET_GPRODD; | ||
36 | if (pick) { | ||
37 | r = ra_restore(as, regcost_ref(as->cost[rset_picktop(pick)-1])); | ||
38 | } else { | ||
39 | r = ra_evict(as, allow & (allow >> 1) & RSET_GPREVEN); | ||
40 | ra_restore(as, regcost_ref(as->cost[r+1])); | ||
41 | } | ||
42 | } | ||
43 | } | ||
44 | lua_assert(rset_test(RSET_GPREVEN, r)); | ||
45 | ra_modified(as, r); | ||
46 | ra_modified(as, r+1); | ||
47 | RA_DBGX((as, "scratchpair $r $r", r, r+1)); | ||
48 | return r; | ||
49 | } | ||
50 | |||
51 | /* -- Guard handling ------------------------------------------------------ */ | ||
52 | |||
53 | /* Generate an exit stub group at the bottom of the reserved MCode memory. */ | ||
54 | static MCode *asm_exitstub_gen(ASMState *as, ExitNo group) | ||
55 | { | ||
56 | MCode *mxp = as->mcbot; | ||
57 | int i; | ||
58 | if (mxp + 4*4+4*EXITSTUBS_PER_GROUP >= as->mctop) | ||
59 | asm_mclimit(as); | ||
60 | /* str lr, [sp]; bl ->vm_exit_handler; .long DISPATCH_address, group. */ | ||
61 | *mxp++ = ARMI_STR|ARMI_LS_P|ARMI_LS_U|ARMF_D(RID_LR)|ARMF_N(RID_SP); | ||
62 | *mxp = ARMI_BL|((((MCode *)(void *)lj_vm_exit_handler-mxp)-2)&0x00ffffffu); | ||
63 | mxp++; | ||
64 | *mxp++ = (MCode)i32ptr(J2GG(as->J)->dispatch); /* DISPATCH address */ | ||
65 | *mxp++ = group*EXITSTUBS_PER_GROUP; | ||
66 | for (i = 0; i < EXITSTUBS_PER_GROUP; i++) | ||
67 | *mxp++ = ARMI_B|((-6-i)&0x00ffffffu); | ||
68 | lj_mcode_commitbot(as->J, mxp); | ||
69 | as->mcbot = mxp; | ||
70 | as->mclim = as->mcbot + MCLIM_REDZONE; | ||
71 | return mxp - EXITSTUBS_PER_GROUP; | ||
72 | } | ||
73 | |||
74 | /* Setup all needed exit stubs. */ | ||
75 | static void asm_exitstub_setup(ASMState *as, ExitNo nexits) | ||
76 | { | ||
77 | ExitNo i; | ||
78 | if (nexits >= EXITSTUBS_PER_GROUP*LJ_MAX_EXITSTUBGR) | ||
79 | lj_trace_err(as->J, LJ_TRERR_SNAPOV); | ||
80 | for (i = 0; i < (nexits+EXITSTUBS_PER_GROUP-1)/EXITSTUBS_PER_GROUP; i++) | ||
81 | if (as->J->exitstubgroup[i] == NULL) | ||
82 | as->J->exitstubgroup[i] = asm_exitstub_gen(as, i); | ||
83 | } | ||
84 | |||
85 | /* Emit conditional branch to exit for guard. */ | ||
86 | static void asm_guardcc(ASMState *as, ARMCC cc) | ||
87 | { | ||
88 | MCode *target = exitstub_addr(as->J, as->snapno); | ||
89 | MCode *p = as->mcp; | ||
90 | if (LJ_UNLIKELY(p == as->invmcp)) { | ||
91 | as->loopinv = 1; | ||
92 | *p = ARMI_BL | ((target-p-2) & 0x00ffffffu); | ||
93 | emit_branch(as, ARMF_CC(ARMI_B, cc^1), p+1); | ||
94 | return; | ||
95 | } | ||
96 | emit_branch(as, ARMF_CC(ARMI_BL, cc), target); | ||
97 | } | ||
98 | |||
99 | /* -- Operand fusion ------------------------------------------------------ */ | ||
100 | |||
101 | /* Limit linear search to this distance. Avoids O(n^2) behavior. */ | ||
102 | #define CONFLICT_SEARCH_LIM 31 | ||
103 | |||
104 | /* Check if there's no conflicting instruction between curins and ref. */ | ||
105 | static int noconflict(ASMState *as, IRRef ref, IROp conflict) | ||
106 | { | ||
107 | IRIns *ir = as->ir; | ||
108 | IRRef i = as->curins; | ||
109 | if (i > ref + CONFLICT_SEARCH_LIM) | ||
110 | return 0; /* Give up, ref is too far away. */ | ||
111 | while (--i > ref) | ||
112 | if (ir[i].o == conflict) | ||
113 | return 0; /* Conflict found. */ | ||
114 | return 1; /* Ok, no conflict. */ | ||
115 | } | ||
116 | |||
117 | /* Fuse the array base of colocated arrays. */ | ||
118 | static int32_t asm_fuseabase(ASMState *as, IRRef ref) | ||
119 | { | ||
120 | IRIns *ir = IR(ref); | ||
121 | if (ir->o == IR_TNEW && ir->op1 <= LJ_MAX_COLOSIZE && | ||
122 | !neverfuse(as) && noconflict(as, ref, IR_NEWREF)) | ||
123 | return (int32_t)sizeof(GCtab); | ||
124 | return 0; | ||
125 | } | ||
126 | |||
127 | /* Fuse array/hash/upvalue reference into register+offset operand. */ | ||
128 | static Reg asm_fuseahuref(ASMState *as, IRRef ref, int32_t *ofsp, RegSet allow) | ||
129 | { | ||
130 | IRIns *ir = IR(ref); | ||
131 | if (ra_noreg(ir->r)) { | ||
132 | if (ir->o == IR_AREF) { | ||
133 | if (mayfuse(as, ref)) { | ||
134 | if (irref_isk(ir->op2)) { | ||
135 | IRRef tab = IR(ir->op1)->op1; | ||
136 | int32_t ofs = asm_fuseabase(as, tab); | ||
137 | IRRef refa = ofs ? tab : ir->op1; | ||
138 | ofs += 8*IR(ir->op2)->i; | ||
139 | if (ofs > -4096 && ofs < 4096) { | ||
140 | *ofsp = ofs; | ||
141 | return ra_alloc1(as, refa, allow); | ||
142 | } | ||
143 | } | ||
144 | } | ||
145 | } else if (ir->o == IR_HREFK) { | ||
146 | if (mayfuse(as, ref)) { | ||
147 | int32_t ofs = (int32_t)(IR(ir->op2)->op2 * sizeof(Node)); | ||
148 | if (ofs < 4096) { | ||
149 | *ofsp = ofs; | ||
150 | return ra_alloc1(as, ir->op1, allow); | ||
151 | } | ||
152 | } | ||
153 | } else if (ir->o == IR_UREFC) { | ||
154 | if (irref_isk(ir->op1)) { | ||
155 | GCfunc *fn = ir_kfunc(IR(ir->op1)); | ||
156 | int32_t ofs = i32ptr(&gcref(fn->l.uvptr[(ir->op2 >> 8)])->uv.tv); | ||
157 | *ofsp = (ofs & 255); /* Mask out less bits to allow LDRD. */ | ||
158 | return ra_allock(as, (ofs & ~255), allow); | ||
159 | } | ||
160 | } | ||
161 | } | ||
162 | *ofsp = 0; | ||
163 | return ra_alloc1(as, ref, allow); | ||
164 | } | ||
165 | |||
166 | /* Fuse m operand into arithmetic/logic instructions. */ | ||
167 | static uint32_t asm_fuseopm(ASMState *as, ARMIns ai, IRRef ref, RegSet allow) | ||
168 | { | ||
169 | IRIns *ir = IR(ref); | ||
170 | if (ra_hasreg(ir->r)) { | ||
171 | ra_noweak(as, ir->r); | ||
172 | return ARMF_M(ir->r); | ||
173 | } else if (irref_isk(ref)) { | ||
174 | uint32_t k = emit_isk12(ai, ir->i); | ||
175 | if (k) | ||
176 | return k; | ||
177 | } else if (mayfuse(as, ref)) { | ||
178 | if (ir->o >= IR_BSHL && ir->o <= IR_BROR) { | ||
179 | Reg m = ra_alloc1(as, ir->op1, allow); | ||
180 | ARMShift sh = ir->o == IR_BSHL ? ARMSH_LSL : | ||
181 | ir->o == IR_BSHR ? ARMSH_LSR : | ||
182 | ir->o == IR_BSAR ? ARMSH_ASR : ARMSH_ROR; | ||
183 | if (irref_isk(ir->op2)) { | ||
184 | return m | ARMF_SH(sh, (IR(ir->op2)->i & 31)); | ||
185 | } else { | ||
186 | Reg s = ra_alloc1(as, ir->op2, rset_exclude(allow, m)); | ||
187 | return m | ARMF_RSH(sh, s); | ||
188 | } | ||
189 | } else if (ir->o == IR_ADD && ir->op1 == ir->op2) { | ||
190 | Reg m = ra_alloc1(as, ir->op1, allow); | ||
191 | return m | ARMF_SH(ARMSH_LSL, 1); | ||
192 | } | ||
193 | } | ||
194 | return ra_allocref(as, ref, allow); | ||
195 | } | ||
196 | |||
197 | /* Fuse shifts into loads/stores. Only bother with BSHL 2 => lsl #2. */ | ||
198 | static IRRef asm_fuselsl2(ASMState *as, IRRef ref) | ||
199 | { | ||
200 | IRIns *ir = IR(ref); | ||
201 | if (ra_noreg(ir->r) && mayfuse(as, ref) && ir->o == IR_BSHL && | ||
202 | irref_isk(ir->op2) && IR(ir->op2)->i == 2) | ||
203 | return ir->op1; | ||
204 | return 0; /* No fusion. */ | ||
205 | } | ||
206 | |||
207 | /* Fuse XLOAD/XSTORE reference into load/store operand. */ | ||
208 | static void asm_fusexref(ASMState *as, ARMIns ai, Reg rd, IRRef ref, | ||
209 | RegSet allow) | ||
210 | { | ||
211 | IRIns *ir = IR(ref); | ||
212 | int32_t ofs = 0; | ||
213 | Reg base; | ||
214 | if (ra_noreg(ir->r) && mayfuse(as, ref)) { | ||
215 | int32_t lim = (ai & 0x04000000) ? 4096 : 256; | ||
216 | if (ir->o == IR_ADD) { | ||
217 | if (irref_isk(ir->op2) && (ofs = IR(ir->op2)->i) > -lim && ofs < lim) { | ||
218 | ref = ir->op1; | ||
219 | } else { | ||
220 | IRRef lref = ir->op1, rref = ir->op2; | ||
221 | Reg rn, rm; | ||
222 | if ((ai & 0x04000000)) { | ||
223 | IRRef sref = asm_fuselsl2(as, rref); | ||
224 | if (sref) { | ||
225 | rref = sref; | ||
226 | ai |= ARMF_SH(ARMSH_LSL, 2); | ||
227 | } else if ((sref = asm_fuselsl2(as, lref)) != 0) { | ||
228 | lref = rref; | ||
229 | rref = sref; | ||
230 | ai |= ARMF_SH(ARMSH_LSL, 2); | ||
231 | } | ||
232 | } | ||
233 | rn = ra_alloc1(as, lref, allow); | ||
234 | rm = ra_alloc1(as, rref, rset_exclude(allow, rn)); | ||
235 | if ((ai & 0x04000000)) ai |= ARMI_LS_R; | ||
236 | emit_dnm(as, ai|ARMI_LS_P|ARMI_LS_U, rd, rn, rm); | ||
237 | return; | ||
238 | } | ||
239 | } else if (ir->o == IR_STRREF) { | ||
240 | ofs = (int32_t)sizeof(GCstr); | ||
241 | if (irref_isk(ir->op2)) { | ||
242 | ofs += IR(ir->op2)->i; | ||
243 | ref = ir->op1; | ||
244 | } else if (irref_isk(ir->op1)) { | ||
245 | ofs += IR(ir->op1)->i; | ||
246 | ref = ir->op2; | ||
247 | } else { | ||
248 | /* NYI: Fuse ADD with constant. */ | ||
249 | Reg rn = ra_alloc1(as, ir->op1, allow); | ||
250 | uint32_t m = asm_fuseopm(as, 0, ir->op2, rset_exclude(allow, rn)); | ||
251 | if ((ai & 0x04000000)) | ||
252 | emit_lso(as, ai, rd, rd, ofs); | ||
253 | else | ||
254 | emit_lsox(as, ai, rd, rd, ofs); | ||
255 | emit_dn(as, ARMI_ADD^m, rd, rn); | ||
256 | return; | ||
257 | } | ||
258 | if (ofs <= -lim || ofs >= lim) { | ||
259 | Reg rn = ra_alloc1(as, ref, allow); | ||
260 | Reg rm = ra_allock(as, ofs, rset_exclude(allow, rn)); | ||
261 | if ((ai & 0x04000000)) ai |= ARMI_LS_R; | ||
262 | emit_dnm(as, ai|ARMI_LS_P|ARMI_LS_U, rd, rn, rm); | ||
263 | return; | ||
264 | } | ||
265 | } | ||
266 | } | ||
267 | base = ra_alloc1(as, ref, allow); | ||
268 | if ((ai & 0x04000000)) | ||
269 | emit_lso(as, ai, rd, base, ofs); | ||
270 | else | ||
271 | emit_lsox(as, ai, rd, base, ofs); | ||
272 | } | ||
273 | |||
274 | /* -- Calls --------------------------------------------------------------- */ | ||
275 | |||
276 | /* Generate a call to a C function. */ | ||
277 | static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args) | ||
278 | { | ||
279 | uint32_t n, nargs = CCI_NARGS(ci); | ||
280 | int32_t ofs = 0; | ||
281 | Reg gpr = REGARG_FIRSTGPR; | ||
282 | if ((void *)ci->func) | ||
283 | emit_call(as, (void *)ci->func); | ||
284 | for (n = 0; n < nargs; n++) { /* Setup args. */ | ||
285 | IRRef ref = args[n]; | ||
286 | IRIns *ir = IR(ref); | ||
287 | if (gpr <= REGARG_LASTGPR) { | ||
288 | lua_assert(rset_test(as->freeset, gpr)); /* Must have been evicted. */ | ||
289 | if (ref) ra_leftov(as, gpr, ref); | ||
290 | gpr++; | ||
291 | } else { | ||
292 | if (ref) { | ||
293 | Reg r = ra_alloc1(as, ref, RSET_GPR); | ||
294 | emit_spstore(as, ir, r, ofs); | ||
295 | } | ||
296 | ofs += 4; | ||
297 | } | ||
298 | } | ||
299 | } | ||
300 | |||
301 | /* Setup result reg/sp for call. Evict scratch regs. */ | ||
302 | static void asm_setupresult(ASMState *as, IRIns *ir, const CCallInfo *ci) | ||
303 | { | ||
304 | RegSet drop = RSET_SCRATCH; | ||
305 | int hiop = ((ir+1)->o == IR_HIOP); | ||
306 | if (ra_hasreg(ir->r)) | ||
307 | rset_clear(drop, ir->r); /* Dest reg handled below. */ | ||
308 | if (hiop && ra_hasreg((ir+1)->r)) | ||
309 | rset_clear(drop, (ir+1)->r); /* Dest reg handled below. */ | ||
310 | ra_evictset(as, drop); /* Evictions must be performed first. */ | ||
311 | if (ra_used(ir)) { | ||
312 | lua_assert(!irt_ispri(ir->t)); | ||
313 | if (hiop) | ||
314 | ra_destpair(as, ir); | ||
315 | else | ||
316 | ra_destreg(as, ir, RID_RET); | ||
317 | } | ||
318 | UNUSED(ci); | ||
319 | } | ||
320 | |||
321 | static void asm_call(ASMState *as, IRIns *ir) | ||
322 | { | ||
323 | IRRef args[CCI_NARGS_MAX]; | ||
324 | const CCallInfo *ci = &lj_ir_callinfo[ir->op2]; | ||
325 | asm_collectargs(as, ir, ci, args); | ||
326 | asm_setupresult(as, ir, ci); | ||
327 | asm_gencall(as, ci, args); | ||
328 | } | ||
329 | |||
330 | static void asm_callx(ASMState *as, IRIns *ir) | ||
331 | { | ||
332 | IRRef args[CCI_NARGS_MAX]; | ||
333 | CCallInfo ci; | ||
334 | IRRef func; | ||
335 | IRIns *irf; | ||
336 | ci.flags = asm_callx_flags(as, ir); | ||
337 | asm_collectargs(as, ir, &ci, args); | ||
338 | asm_setupresult(as, ir, &ci); | ||
339 | func = ir->op2; irf = IR(func); | ||
340 | if (irf->o == IR_CARG) { func = irf->op1; irf = IR(func); } | ||
341 | if (irref_isk(func)) { /* Call to constant address. */ | ||
342 | ci.func = (ASMFunction)(void *)(irf->i); | ||
343 | } else { /* Need a non-argument register for indirect calls. */ | ||
344 | Reg freg = ra_alloc1(as, func, RSET_RANGE(RID_R4, RID_R12+1)); | ||
345 | emit_m(as, ARMI_BLXr, freg); | ||
346 | ci.func = (ASMFunction)(void *)0; | ||
347 | } | ||
348 | asm_gencall(as, &ci, args); | ||
349 | } | ||
350 | |||
351 | /* -- Returns ------------------------------------------------------------- */ | ||
352 | |||
353 | /* Return to lower frame. Guard that it goes to the right spot. */ | ||
354 | static void asm_retf(ASMState *as, IRIns *ir) | ||
355 | { | ||
356 | Reg base = ra_alloc1(as, REF_BASE, RSET_GPR); | ||
357 | void *pc = ir_kptr(IR(ir->op2)); | ||
358 | int32_t delta = 1+bc_a(*((const BCIns *)pc - 1)); | ||
359 | as->topslot -= (BCReg)delta; | ||
360 | if ((int32_t)as->topslot < 0) as->topslot = 0; | ||
361 | /* Need to force a spill on REF_BASE now to update the stack slot. */ | ||
362 | emit_lso(as, ARMI_STR, base, RID_SP, ra_spill(as, IR(REF_BASE))); | ||
363 | emit_setgl(as, base, jit_base); | ||
364 | emit_addptr(as, base, -8*delta); | ||
365 | asm_guardcc(as, CC_NE); | ||
366 | emit_nm(as, ARMI_CMP, RID_TMP, | ||
367 | ra_allock(as, i32ptr(pc), rset_exclude(RSET_GPR, base))); | ||
368 | emit_lso(as, ARMI_LDR, RID_TMP, base, -4); | ||
369 | } | ||
370 | |||
371 | /* -- Type conversions ---------------------------------------------------- */ | ||
372 | |||
373 | static void asm_conv(ASMState *as, IRIns *ir) | ||
374 | { | ||
375 | Reg dest = ra_dest(as, ir, RSET_GPR); | ||
376 | IRType st = (IRType)(ir->op2 & IRCONV_SRCMASK); | ||
377 | /* FP conversions and 64 bit integer conversions are handled by SPLIT. */ | ||
378 | lua_assert(!irt_isfp(ir->t) && !(st == IRT_NUM || st == IRT_FLOAT)); | ||
379 | lua_assert(!irt_isint64(ir->t) && !(st == IRT_I64 || st == IRT_U64)); | ||
380 | if (st >= IRT_I8 && st <= IRT_U16) { /* Extend to 32 bit integer. */ | ||
381 | Reg left = ra_alloc1(as, ir->op1, RSET_GPR); | ||
382 | lua_assert(irt_isint(ir->t) || irt_isu32(ir->t)); | ||
383 | if ((as->flags & JIT_F_ARMV6)) { | ||
384 | ARMIns ai = st == IRT_I8 ? ARMI_SXTB : | ||
385 | st == IRT_U8 ? ARMI_UXTB : | ||
386 | st == IRT_I16 ? ARMI_SXTH : ARMI_UXTH; | ||
387 | emit_dm(as, ai, dest, left); | ||
388 | } else if (st == IRT_U8) { | ||
389 | emit_dn(as, ARMI_AND|ARMI_K12|255, dest, left); | ||
390 | } else { | ||
391 | uint32_t shift = st == IRT_I8 ? 24 : 16; | ||
392 | ARMShift sh = st == IRT_U16 ? ARMSH_LSR : ARMSH_ASR; | ||
393 | emit_dm(as, ARMI_MOV|ARMF_SH(sh, shift), dest, RID_TMP); | ||
394 | emit_dm(as, ARMI_MOV|ARMF_SH(ARMSH_LSL, shift), RID_TMP, left); | ||
395 | } | ||
396 | } else { /* Handle 32/32 bit no-op (cast). */ | ||
397 | ra_leftov(as, dest, ir->op1); /* Do nothing, but may need to move regs. */ | ||
398 | } | ||
399 | } | ||
400 | |||
401 | static void asm_strto(ASMState *as, IRIns *ir) | ||
402 | { | ||
403 | const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_str_tonum]; | ||
404 | IRRef args[2]; | ||
405 | Reg rlo = 0, rhi = 0, tmp; | ||
406 | int destused = ra_used(ir); | ||
407 | int32_t ofs = 0; | ||
408 | ra_evictset(as, RSET_SCRATCH); | ||
409 | if (destused) { | ||
410 | if (ra_hasspill(ir->s) && ra_hasspill((ir+1)->s) && | ||
411 | (ir->s & 1) == 0 && ir->s + 1 == (ir+1)->s) { | ||
412 | int i; | ||
413 | for (i = 0; i < 2; i++) { | ||
414 | Reg r = (ir+i)->r; | ||
415 | if (ra_hasreg(r)) { | ||
416 | ra_free(as, r); | ||
417 | ra_modified(as, r); | ||
418 | emit_spload(as, ir+i, r, sps_scale((ir+i)->s)); | ||
419 | } | ||
420 | } | ||
421 | ofs = sps_scale(ir->s); | ||
422 | destused = 0; | ||
423 | } else { | ||
424 | rhi = ra_dest(as, ir+1, RSET_GPR); | ||
425 | rlo = ra_dest(as, ir, rset_exclude(RSET_GPR, rhi)); | ||
426 | } | ||
427 | } | ||
428 | asm_guardcc(as, CC_EQ); | ||
429 | if (destused) { | ||
430 | emit_lso(as, ARMI_LDR, rhi, RID_SP, 4); | ||
431 | emit_lso(as, ARMI_LDR, rlo, RID_SP, 0); | ||
432 | } | ||
433 | emit_n(as, ARMI_CMP|ARMI_K12|0, RID_RET); /* Test return status. */ | ||
434 | args[0] = ir->op1; /* GCstr *str */ | ||
435 | args[1] = ASMREF_TMP1; /* TValue *n */ | ||
436 | asm_gencall(as, ci, args); | ||
437 | tmp = ra_releasetmp(as, ASMREF_TMP1); | ||
438 | if (ofs == 0) | ||
439 | emit_dm(as, ARMI_MOV, tmp, RID_SP); | ||
440 | else | ||
441 | emit_opk(as, ARMI_ADD, tmp, RID_SP, ofs, RSET_GPR); | ||
442 | } | ||
443 | |||
444 | /* Get pointer to TValue. */ | ||
445 | static void asm_tvptr(ASMState *as, Reg dest, IRRef ref) | ||
446 | { | ||
447 | IRIns *ir = IR(ref); | ||
448 | if (irt_isnum(ir->t)) { /* Use the number constant itself as a TValue. */ | ||
449 | lua_assert(irref_isk(ref)); | ||
450 | ra_allockreg(as, i32ptr(ir_knum(ir)), dest); | ||
451 | } else { | ||
452 | /* Otherwise use [sp] and [sp+4] to hold the TValue. */ | ||
453 | RegSet allow = rset_exclude(RSET_GPR, dest); | ||
454 | Reg type; | ||
455 | emit_dm(as, ARMI_MOV, dest, RID_SP); | ||
456 | if (!irt_ispri(ir->t)) { | ||
457 | Reg src = ra_alloc1(as, ref, allow); | ||
458 | emit_lso(as, ARMI_STR, src, RID_SP, 0); | ||
459 | } | ||
460 | if ((ir+1)->o == IR_HIOP) | ||
461 | type = ra_alloc1(as, ref+1, allow); | ||
462 | else | ||
463 | type = ra_allock(as, irt_toitype(ir->t), allow); | ||
464 | emit_lso(as, ARMI_STR, type, RID_SP, 4); | ||
465 | } | ||
466 | } | ||
467 | |||
468 | static void asm_tostr(ASMState *as, IRIns *ir) | ||
469 | { | ||
470 | IRRef args[2]; | ||
471 | args[0] = ASMREF_L; | ||
472 | as->gcsteps++; | ||
473 | if (irt_isnum(IR(ir->op1)->t) || (ir+1)->o == IR_HIOP) { | ||
474 | const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_str_fromnum]; | ||
475 | args[1] = ASMREF_TMP1; /* const lua_Number * */ | ||
476 | asm_setupresult(as, ir, ci); /* GCstr * */ | ||
477 | asm_gencall(as, ci, args); | ||
478 | asm_tvptr(as, ra_releasetmp(as, ASMREF_TMP1), ir->op1); | ||
479 | } else { | ||
480 | const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_str_fromint]; | ||
481 | args[1] = ir->op1; /* int32_t k */ | ||
482 | asm_setupresult(as, ir, ci); /* GCstr * */ | ||
483 | asm_gencall(as, ci, args); | ||
484 | } | ||
485 | } | ||
486 | |||
487 | /* -- Memory references --------------------------------------------------- */ | ||
488 | |||
489 | static void asm_aref(ASMState *as, IRIns *ir) | ||
490 | { | ||
491 | Reg dest = ra_dest(as, ir, RSET_GPR); | ||
492 | Reg idx, base; | ||
493 | if (irref_isk(ir->op2)) { | ||
494 | IRRef tab = IR(ir->op1)->op1; | ||
495 | int32_t ofs = asm_fuseabase(as, tab); | ||
496 | IRRef refa = ofs ? tab : ir->op1; | ||
497 | uint32_t k = emit_isk12(ARMI_ADD, ofs + 8*IR(ir->op2)->i); | ||
498 | if (k) { | ||
499 | base = ra_alloc1(as, refa, RSET_GPR); | ||
500 | emit_dn(as, ARMI_ADD^k, dest, base); | ||
501 | return; | ||
502 | } | ||
503 | } | ||
504 | base = ra_alloc1(as, ir->op1, RSET_GPR); | ||
505 | idx = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, base)); | ||
506 | emit_dnm(as, ARMI_ADD|ARMF_SH(ARMSH_LSL, 3), dest, base, idx); | ||
507 | } | ||
508 | |||
509 | /* Inlined hash lookup. Specialized for key type and for const keys. | ||
510 | ** The equivalent C code is: | ||
511 | ** Node *n = hashkey(t, key); | ||
512 | ** do { | ||
513 | ** if (lj_obj_equal(&n->key, key)) return &n->val; | ||
514 | ** } while ((n = nextnode(n))); | ||
515 | ** return niltv(L); | ||
516 | */ | ||
517 | static void asm_href(ASMState *as, IRIns *ir, IROp merge) | ||
518 | { | ||
519 | RegSet allow = RSET_GPR; | ||
520 | int destused = ra_used(ir); | ||
521 | Reg dest = ra_dest(as, ir, allow); | ||
522 | Reg tab = ra_alloc1(as, ir->op1, rset_clear(allow, dest)); | ||
523 | Reg key = 0, keyhi = 0, keynumhi = RID_NONE, tmp = RID_TMP; | ||
524 | IRRef refkey = ir->op2; | ||
525 | IRIns *irkey = IR(refkey); | ||
526 | IRType1 kt = irkey->t; | ||
527 | int32_t k = 0, khi = emit_isk12(ARMI_CMP, irt_toitype(kt)); | ||
528 | uint32_t khash; | ||
529 | MCLabel l_end, l_loop; | ||
530 | rset_clear(allow, tab); | ||
531 | if (!irref_isk(refkey) || irt_isstr(kt)) { | ||
532 | key = ra_alloc1(as, refkey, allow); | ||
533 | rset_clear(allow, key); | ||
534 | if (irkey[1].o == IR_HIOP) { | ||
535 | if (ra_hasreg((irkey+1)->r)) { | ||
536 | keynumhi = (irkey+1)->r; | ||
537 | keyhi = RID_TMP; | ||
538 | ra_noweak(as, keynumhi); | ||
539 | } else { | ||
540 | keyhi = keynumhi = ra_allocref(as, refkey+1, allow); | ||
541 | } | ||
542 | rset_clear(allow, keynumhi); | ||
543 | khi = 0; | ||
544 | } | ||
545 | } else if (irt_isnum(kt)) { | ||
546 | int32_t val = (int32_t)ir_knum(irkey)->u32.lo; | ||
547 | k = emit_isk12(ARMI_CMP, val); | ||
548 | if (!k) { | ||
549 | key = ra_allock(as, val, allow); | ||
550 | rset_clear(allow, key); | ||
551 | } | ||
552 | val = (int32_t)ir_knum(irkey)->u32.hi; | ||
553 | khi = emit_isk12(ARMI_CMP, val); | ||
554 | if (!khi) { | ||
555 | keyhi = ra_allock(as, val, allow); | ||
556 | rset_clear(allow, keyhi); | ||
557 | } | ||
558 | } else if (!irt_ispri(kt)) { | ||
559 | k = emit_isk12(ARMI_CMP, irkey->i); | ||
560 | if (!k) { | ||
561 | key = ra_alloc1(as, refkey, allow); | ||
562 | rset_clear(allow, key); | ||
563 | } | ||
564 | } | ||
565 | if (!irt_ispri(kt)) | ||
566 | tmp = ra_scratchpair(as, allow); | ||
567 | |||
568 | /* Key not found in chain: jump to exit (if merged) or load niltv. */ | ||
569 | l_end = emit_label(as); | ||
570 | as->invmcp = NULL; | ||
571 | if (merge == IR_NE) | ||
572 | asm_guardcc(as, CC_AL); | ||
573 | else if (destused) | ||
574 | emit_loada(as, dest, niltvg(J2G(as->J))); | ||
575 | |||
576 | /* Follow hash chain until the end. */ | ||
577 | l_loop = --as->mcp; | ||
578 | emit_n(as, ARMI_CMP|ARMI_K12|0, dest); | ||
579 | emit_lso(as, ARMI_LDR, dest, dest, (int32_t)offsetof(Node, next)); | ||
580 | |||
581 | /* Type and value comparison. */ | ||
582 | if (merge == IR_EQ) | ||
583 | asm_guardcc(as, CC_EQ); | ||
584 | else | ||
585 | emit_branch(as, ARMF_CC(ARMI_B, CC_EQ), l_end); | ||
586 | if (!irt_ispri(kt)) { | ||
587 | emit_nm(as, ARMF_CC(ARMI_CMP, CC_EQ)^khi, tmp+1, keyhi); | ||
588 | emit_nm(as, ARMI_CMP^k, tmp, key); | ||
589 | emit_lsox(as, ARMI_LDRD, tmp, dest, (int32_t)offsetof(Node, key)); | ||
590 | } else { | ||
591 | emit_n(as, ARMI_CMP^khi, tmp); | ||
592 | emit_lso(as, ARMI_LDR, tmp, dest, (int32_t)offsetof(Node, key.it)); | ||
593 | } | ||
594 | *l_loop = ARMF_CC(ARMI_B, CC_NE) | ((as->mcp-l_loop-2) & 0x00ffffffu); | ||
595 | |||
596 | /* Load main position relative to tab->node into dest. */ | ||
597 | khash = irref_isk(refkey) ? ir_khash(irkey) : 1; | ||
598 | if (khash == 0) { | ||
599 | emit_lso(as, ARMI_LDR, dest, tab, (int32_t)offsetof(GCtab, node)); | ||
600 | } else { | ||
601 | emit_dnm(as, ARMI_ADD|ARMF_SH(ARMSH_LSL, 3), dest, dest, tmp); | ||
602 | emit_dnm(as, ARMI_ADD|ARMF_SH(ARMSH_LSL, 1), tmp, tmp, tmp); | ||
603 | if (irt_isstr(kt)) { /* Fetch of str->hash is cheaper than ra_allock. */ | ||
604 | emit_dnm(as, ARMI_AND, tmp, tmp+1, RID_TMP); | ||
605 | emit_lso(as, ARMI_LDR, dest, tab, (int32_t)offsetof(GCtab, node)); | ||
606 | emit_lso(as, ARMI_LDR, tmp+1, key, (int32_t)offsetof(GCstr, hash)); | ||
607 | emit_lso(as, ARMI_LDR, RID_TMP, tab, (int32_t)offsetof(GCtab, hmask)); | ||
608 | } else if (irref_isk(refkey)) { | ||
609 | emit_opk(as, ARMI_AND, tmp, RID_TMP, (int32_t)khash, | ||
610 | rset_exclude(rset_exclude(RSET_GPR, tab), dest)); | ||
611 | emit_lso(as, ARMI_LDR, dest, tab, (int32_t)offsetof(GCtab, node)); | ||
612 | emit_lso(as, ARMI_LDR, RID_TMP, tab, (int32_t)offsetof(GCtab, hmask)); | ||
613 | } else { /* Must match with hash*() in lj_tab.c. */ | ||
614 | if (ra_hasreg(keynumhi)) { /* Canonicalize +-0.0 to 0.0. */ | ||
615 | if (keyhi == RID_TMP) | ||
616 | emit_dm(as, ARMF_CC(ARMI_MOV, CC_NE), keyhi, keynumhi); | ||
617 | emit_d(as, ARMF_CC(ARMI_MOV, CC_EQ)|ARMI_K12|0, keyhi); | ||
618 | } | ||
619 | emit_dnm(as, ARMI_AND, tmp, tmp, RID_TMP); | ||
620 | emit_dnm(as, ARMI_SUB|ARMF_SH(ARMSH_ROR, 32-HASH_ROT3), tmp, tmp, tmp+1); | ||
621 | emit_lso(as, ARMI_LDR, dest, tab, (int32_t)offsetof(GCtab, node)); | ||
622 | emit_dnm(as, ARMI_EOR|ARMF_SH(ARMSH_ROR, 32-((HASH_ROT2+HASH_ROT1)&31)), | ||
623 | tmp, tmp+1, tmp); | ||
624 | emit_lso(as, ARMI_LDR, RID_TMP, tab, (int32_t)offsetof(GCtab, hmask)); | ||
625 | emit_dnm(as, ARMI_SUB|ARMF_SH(ARMSH_ROR, 32-HASH_ROT1), tmp+1, tmp+1, tmp); | ||
626 | if (ra_hasreg(keynumhi)) { | ||
627 | emit_dnm(as, ARMI_EOR, tmp+1, tmp, key); | ||
628 | emit_dnm(as, ARMI_ORR|ARMI_S, RID_TMP, tmp, key); /* Test for +-0.0. */ | ||
629 | emit_dnm(as, ARMI_ADD, tmp, keynumhi, keynumhi); | ||
630 | } else { | ||
631 | emit_dnm(as, ARMI_EOR, tmp+1, tmp, key); | ||
632 | emit_opk(as, ARMI_ADD, tmp, key, (int32_t)HASH_BIAS, | ||
633 | rset_exclude(rset_exclude(RSET_GPR, tab), key)); | ||
634 | } | ||
635 | } | ||
636 | } | ||
637 | } | ||
638 | |||
639 | static void asm_hrefk(ASMState *as, IRIns *ir) | ||
640 | { | ||
641 | IRIns *kslot = IR(ir->op2); | ||
642 | IRIns *irkey = IR(kslot->op1); | ||
643 | int32_t ofs = (int32_t)(kslot->op2 * sizeof(Node)); | ||
644 | int32_t kofs = ofs + (int32_t)offsetof(Node, key); | ||
645 | Reg dest = (ra_used(ir) || ofs > 4095) ? ra_dest(as, ir, RSET_GPR) : RID_NONE; | ||
646 | Reg node = ra_alloc1(as, ir->op1, RSET_GPR); | ||
647 | Reg key = RID_NONE, type = RID_TMP, idx = node; | ||
648 | RegSet allow = rset_exclude(RSET_GPR, node); | ||
649 | lua_assert(ofs % sizeof(Node) == 0); | ||
650 | if (ofs > 4095) { | ||
651 | idx = dest; | ||
652 | rset_clear(allow, dest); | ||
653 | kofs = (int32_t)offsetof(Node, key); | ||
654 | } else if (ra_hasreg(dest)) { | ||
655 | emit_opk(as, ARMI_ADD, dest, node, ofs, allow); | ||
656 | } | ||
657 | asm_guardcc(as, CC_NE); | ||
658 | if (!irt_ispri(irkey->t)) { | ||
659 | RegSet even = (as->freeset & (as->freeset >> 1) & allow & RSET_GPREVEN); | ||
660 | if (even) { | ||
661 | key = ra_scratch(as, even); | ||
662 | if (rset_test(as->freeset, key+1)) { | ||
663 | type = key+1; | ||
664 | ra_modified(as, type); | ||
665 | } | ||
666 | } else { | ||
667 | key = ra_scratch(as, allow); | ||
668 | } | ||
669 | rset_clear(allow, key); | ||
670 | } | ||
671 | rset_clear(allow, type); | ||
672 | if (irt_isnum(irkey->t)) { | ||
673 | emit_opk(as, ARMF_CC(ARMI_CMP, CC_EQ), 0, type, | ||
674 | (int32_t)ir_knum(irkey)->u32.hi, allow); | ||
675 | emit_opk(as, ARMI_CMP, 0, key, | ||
676 | (int32_t)ir_knum(irkey)->u32.lo, allow); | ||
677 | } else if (ra_hasreg(key)) { | ||
678 | emit_n(as, ARMF_CC(ARMI_CMN, CC_EQ)|ARMI_K12|-irt_toitype(irkey->t), type); | ||
679 | emit_opk(as, ARMI_CMP, 0, key, irkey->i, allow); | ||
680 | } else { | ||
681 | emit_n(as, ARMI_CMN|ARMI_K12|-irt_toitype(irkey->t), type); | ||
682 | } | ||
683 | emit_lso(as, ARMI_LDR, type, idx, kofs+4); | ||
684 | if (ra_hasreg(key)) emit_lso(as, ARMI_LDR, key, idx, kofs); | ||
685 | if (ofs > 4095) | ||
686 | emit_opk(as, ARMI_ADD, dest, node, ofs, RSET_GPR); | ||
687 | } | ||
688 | |||
689 | static void asm_newref(ASMState *as, IRIns *ir) | ||
690 | { | ||
691 | const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_tab_newkey]; | ||
692 | IRRef args[3]; | ||
693 | args[0] = ASMREF_L; /* lua_State *L */ | ||
694 | args[1] = ir->op1; /* GCtab *t */ | ||
695 | args[2] = ASMREF_TMP1; /* cTValue *key */ | ||
696 | asm_setupresult(as, ir, ci); /* TValue * */ | ||
697 | asm_gencall(as, ci, args); | ||
698 | asm_tvptr(as, ra_releasetmp(as, ASMREF_TMP1), ir->op2); | ||
699 | } | ||
700 | |||
701 | static void asm_uref(ASMState *as, IRIns *ir) | ||
702 | { | ||
703 | /* NYI: Check that UREFO is still open and not aliasing a slot. */ | ||
704 | Reg dest = ra_dest(as, ir, RSET_GPR); | ||
705 | if (irref_isk(ir->op1)) { | ||
706 | GCfunc *fn = ir_kfunc(IR(ir->op1)); | ||
707 | MRef *v = &gcref(fn->l.uvptr[(ir->op2 >> 8)])->uv.v; | ||
708 | emit_lsptr(as, ARMI_LDR, dest, v); | ||
709 | } else { | ||
710 | Reg uv = ra_scratch(as, RSET_GPR); | ||
711 | Reg func = ra_alloc1(as, ir->op1, RSET_GPR); | ||
712 | if (ir->o == IR_UREFC) { | ||
713 | asm_guardcc(as, CC_NE); | ||
714 | emit_n(as, ARMI_CMP|ARMI_K12|1, RID_TMP); | ||
715 | emit_opk(as, ARMI_ADD, dest, uv, | ||
716 | (int32_t)offsetof(GCupval, tv), RSET_GPR); | ||
717 | emit_lso(as, ARMI_LDRB, RID_TMP, uv, (int32_t)offsetof(GCupval, closed)); | ||
718 | } else { | ||
719 | emit_lso(as, ARMI_LDR, dest, uv, (int32_t)offsetof(GCupval, v)); | ||
720 | } | ||
721 | emit_lso(as, ARMI_LDR, uv, func, | ||
722 | (int32_t)offsetof(GCfuncL, uvptr) + 4*(int32_t)(ir->op2 >> 8)); | ||
723 | } | ||
724 | } | ||
725 | |||
726 | static void asm_fref(ASMState *as, IRIns *ir) | ||
727 | { | ||
728 | UNUSED(as); UNUSED(ir); | ||
729 | lua_assert(!ra_used(ir)); | ||
730 | } | ||
731 | |||
732 | static void asm_strref(ASMState *as, IRIns *ir) | ||
733 | { | ||
734 | Reg dest = ra_dest(as, ir, RSET_GPR); | ||
735 | IRRef ref = ir->op2, refk = ir->op1; | ||
736 | Reg r; | ||
737 | if (irref_isk(ref)) { | ||
738 | IRRef tmp = refk; refk = ref; ref = tmp; | ||
739 | } else if (!irref_isk(refk)) { | ||
740 | uint32_t k, m = ARMI_K12|sizeof(GCstr); | ||
741 | Reg right, left = ra_alloc1(as, ir->op1, RSET_GPR); | ||
742 | IRIns *irr = IR(ir->op2); | ||
743 | if (ra_hasreg(irr->r)) { | ||
744 | ra_noweak(as, irr->r); | ||
745 | right = irr->r; | ||
746 | } else if (mayfuse(as, irr->op2) && | ||
747 | irr->o == IR_ADD && irref_isk(irr->op2) && | ||
748 | (k = emit_isk12(ARMI_ADD, | ||
749 | (int32_t)sizeof(GCstr) + IR(irr->op2)->i))) { | ||
750 | m = k; | ||
751 | right = ra_alloc1(as, irr->op1, rset_exclude(RSET_GPR, left)); | ||
752 | } else { | ||
753 | right = ra_allocref(as, ir->op2, rset_exclude(RSET_GPR, left)); | ||
754 | } | ||
755 | emit_dn(as, ARMI_ADD^m, dest, dest); | ||
756 | emit_dnm(as, ARMI_ADD, dest, left, right); | ||
757 | return; | ||
758 | } | ||
759 | r = ra_alloc1(as, ref, RSET_GPR); | ||
760 | emit_opk(as, ARMI_ADD, dest, r, | ||
761 | sizeof(GCstr) + IR(refk)->i, rset_exclude(RSET_GPR, r)); | ||
762 | } | ||
763 | |||
764 | /* -- Loads and stores ---------------------------------------------------- */ | ||
765 | |||
766 | static ARMIns asm_fxloadins(IRIns *ir) | ||
767 | { | ||
768 | switch (irt_type(ir->t)) { | ||
769 | case IRT_I8: return ARMI_LDRSB; | ||
770 | case IRT_U8: return ARMI_LDRB; | ||
771 | case IRT_I16: return ARMI_LDRSH; | ||
772 | case IRT_U16: return ARMI_LDRH; | ||
773 | case IRT_NUM: lua_assert(0); | ||
774 | case IRT_FLOAT: | ||
775 | default: return ARMI_LDR; | ||
776 | } | ||
777 | } | ||
778 | |||
779 | static ARMIns asm_fxstoreins(IRIns *ir) | ||
780 | { | ||
781 | switch (irt_type(ir->t)) { | ||
782 | case IRT_I8: case IRT_U8: return ARMI_STRB; | ||
783 | case IRT_I16: case IRT_U16: return ARMI_STRH; | ||
784 | case IRT_NUM: lua_assert(0); | ||
785 | case IRT_FLOAT: | ||
786 | default: return ARMI_STR; | ||
787 | } | ||
788 | } | ||
789 | |||
790 | static void asm_fload(ASMState *as, IRIns *ir) | ||
791 | { | ||
792 | Reg dest = ra_dest(as, ir, RSET_GPR); | ||
793 | Reg idx = ra_alloc1(as, ir->op1, RSET_GPR); | ||
794 | ARMIns ai = asm_fxloadins(ir); | ||
795 | int32_t ofs; | ||
796 | if (ir->op2 == IRFL_TAB_ARRAY) { | ||
797 | ofs = asm_fuseabase(as, ir->op1); | ||
798 | if (ofs) { /* Turn the t->array load into an add for colocated arrays. */ | ||
799 | emit_dn(as, ARMI_ADD|ARMI_K12|ofs, dest, idx); | ||
800 | return; | ||
801 | } | ||
802 | } | ||
803 | ofs = field_ofs[ir->op2]; | ||
804 | if ((ai & 0x04000000)) | ||
805 | emit_lso(as, ai, dest, idx, ofs); | ||
806 | else | ||
807 | emit_lsox(as, ai, dest, idx, ofs); | ||
808 | } | ||
809 | |||
810 | static void asm_fstore(ASMState *as, IRIns *ir) | ||
811 | { | ||
812 | Reg src = ra_alloc1(as, ir->op2, RSET_GPR); | ||
813 | IRIns *irf = IR(ir->op1); | ||
814 | Reg idx = ra_alloc1(as, irf->op1, rset_exclude(RSET_GPR, src)); | ||
815 | int32_t ofs = field_ofs[irf->op2]; | ||
816 | ARMIns ai = asm_fxstoreins(ir); | ||
817 | if ((ai & 0x04000000)) | ||
818 | emit_lso(as, ai, src, idx, ofs); | ||
819 | else | ||
820 | emit_lsox(as, ai, src, idx, ofs); | ||
821 | } | ||
822 | |||
823 | static void asm_xload(ASMState *as, IRIns *ir) | ||
824 | { | ||
825 | Reg dest = ra_dest(as, ir, RSET_GPR); | ||
826 | lua_assert(!(ir->op2 & IRXLOAD_UNALIGNED)); | ||
827 | asm_fusexref(as, asm_fxloadins(ir), dest, ir->op1, RSET_GPR); | ||
828 | } | ||
829 | |||
830 | static void asm_xstore(ASMState *as, IRIns *ir) | ||
831 | { | ||
832 | Reg src = ra_alloc1(as, ir->op2, RSET_GPR); | ||
833 | asm_fusexref(as, asm_fxstoreins(ir), src, ir->op1, | ||
834 | rset_exclude(RSET_GPR, src)); | ||
835 | } | ||
836 | |||
837 | static void asm_ahuvload(ASMState *as, IRIns *ir) | ||
838 | { | ||
839 | int hiop = ((ir+1)->o == IR_HIOP); | ||
840 | IRType t = hiop ? IRT_NUM : irt_type(ir->t); | ||
841 | Reg dest = RID_NONE, type = RID_NONE, idx; | ||
842 | RegSet allow = RSET_GPR; | ||
843 | int32_t ofs = 0; | ||
844 | if (hiop && ra_used(ir+1)) { | ||
845 | type = ra_dest(as, ir+1, allow); | ||
846 | rset_clear(allow, type); | ||
847 | } | ||
848 | if (ra_used(ir)) { | ||
849 | lua_assert(irt_isint(ir->t) || irt_isaddr(ir->t)); | ||
850 | dest = ra_dest(as, ir, allow); | ||
851 | rset_clear(allow, dest); | ||
852 | } | ||
853 | idx = asm_fuseahuref(as, ir->op1, &ofs, allow); | ||
854 | if (!hiop || type == RID_NONE) { | ||
855 | rset_clear(allow, idx); | ||
856 | if (ofs < 256 && ra_hasreg(dest) && (dest & 1) == 0 && | ||
857 | rset_test((as->freeset & allow), dest+1)) { | ||
858 | type = dest+1; | ||
859 | ra_modified(as, type); | ||
860 | } else { | ||
861 | type = RID_TMP; | ||
862 | } | ||
863 | } | ||
864 | asm_guardcc(as, t == IRT_NUM ? CC_HS : CC_NE); | ||
865 | emit_n(as, ARMI_CMN|ARMI_K12|-irt_toitype_(t), type); | ||
866 | if (ra_hasreg(dest)) emit_lso(as, ARMI_LDR, dest, idx, ofs); | ||
867 | emit_lso(as, ARMI_LDR, type, idx, ofs+4); | ||
868 | } | ||
869 | |||
870 | static void asm_ahustore(ASMState *as, IRIns *ir) | ||
871 | { | ||
872 | RegSet allow = RSET_GPR; | ||
873 | Reg idx, src = RID_NONE, type = RID_NONE; | ||
874 | int32_t ofs = 0; | ||
875 | int hiop = ((ir+1)->o == IR_HIOP); | ||
876 | if (!irt_ispri(ir->t)) { | ||
877 | src = ra_alloc1(as, ir->op2, allow); | ||
878 | rset_clear(allow, src); | ||
879 | } | ||
880 | if (hiop) | ||
881 | type = ra_alloc1(as, (ir+1)->op2, allow); | ||
882 | else | ||
883 | type = ra_allock(as, (int32_t)irt_toitype(ir->t), allow); | ||
884 | idx = asm_fuseahuref(as, ir->op1, &ofs, rset_exclude(allow, type)); | ||
885 | if (ra_hasreg(src)) emit_lso(as, ARMI_STR, src, idx, ofs); | ||
886 | emit_lso(as, ARMI_STR, type, idx, ofs+4); | ||
887 | } | ||
888 | |||
889 | static void asm_sload(ASMState *as, IRIns *ir) | ||
890 | { | ||
891 | int32_t ofs = 8*((int32_t)ir->op1-1) + ((ir->op2 & IRSLOAD_FRAME) ? 4 : 0); | ||
892 | int hiop = ((ir+1)->o == IR_HIOP); | ||
893 | IRType t = hiop ? IRT_NUM : irt_type(ir->t); | ||
894 | Reg dest = RID_NONE, type = RID_NONE, base; | ||
895 | RegSet allow = RSET_GPR; | ||
896 | lua_assert(!(ir->op2 & IRSLOAD_PARENT)); /* Handled by asm_head_side(). */ | ||
897 | lua_assert(irt_isguard(ir->t) || !(ir->op2 & IRSLOAD_TYPECHECK)); | ||
898 | lua_assert(!(ir->op2 & IRSLOAD_CONVERT)); /* Handled by LJ_SOFTFP SPLIT. */ | ||
899 | if (hiop && ra_used(ir+1)) { | ||
900 | type = ra_dest(as, ir+1, allow); | ||
901 | rset_clear(allow, type); | ||
902 | } | ||
903 | if (ra_used(ir)) { | ||
904 | lua_assert(irt_isint(ir->t) || irt_isaddr(ir->t)); | ||
905 | dest = ra_dest(as, ir, allow); | ||
906 | rset_clear(allow, dest); | ||
907 | } | ||
908 | base = ra_alloc1(as, REF_BASE, allow); | ||
909 | if ((ir->op2 & IRSLOAD_TYPECHECK)) { | ||
910 | if (ra_noreg(type)) { | ||
911 | rset_clear(allow, base); | ||
912 | if (ofs < 256 && ra_hasreg(dest) && (dest & 1) == 0 && | ||
913 | rset_test((as->freeset & allow), dest+1)) { | ||
914 | type = dest+1; | ||
915 | ra_modified(as, type); | ||
916 | } else { | ||
917 | type = RID_TMP; | ||
918 | } | ||
919 | } | ||
920 | asm_guardcc(as, t == IRT_NUM ? CC_HS : CC_NE); | ||
921 | emit_n(as, ARMI_CMN|ARMI_K12|-irt_toitype_(t), type); | ||
922 | } | ||
923 | if (ra_hasreg(dest)) emit_lso(as, ARMI_LDR, dest, base, ofs); | ||
924 | if (ra_hasreg(type)) emit_lso(as, ARMI_LDR, type, base, ofs+4); | ||
925 | } | ||
926 | |||
927 | /* -- Allocations --------------------------------------------------------- */ | ||
928 | |||
929 | #if LJ_HASFFI | ||
930 | static void asm_cnew(ASMState *as, IRIns *ir) | ||
931 | { | ||
932 | CTState *cts = ctype_ctsG(J2G(as->J)); | ||
933 | CTypeID typeid = (CTypeID)IR(ir->op1)->i; | ||
934 | CTSize sz = (ir->o == IR_CNEWI || ir->op2 == REF_NIL) ? | ||
935 | lj_ctype_size(cts, typeid) : (CTSize)IR(ir->op2)->i; | ||
936 | const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_mem_newgco]; | ||
937 | IRRef args[2]; | ||
938 | RegSet allow = (RSET_GPR & ~RSET_SCRATCH); | ||
939 | RegSet drop = RSET_SCRATCH; | ||
940 | lua_assert(sz != CTSIZE_INVALID); | ||
941 | |||
942 | args[0] = ASMREF_L; /* lua_State *L */ | ||
943 | args[1] = ASMREF_TMP1; /* MSize size */ | ||
944 | as->gcsteps++; | ||
945 | |||
946 | if (ra_hasreg(ir->r)) | ||
947 | rset_clear(drop, ir->r); /* Dest reg handled below. */ | ||
948 | ra_evictset(as, drop); | ||
949 | if (ra_used(ir)) | ||
950 | ra_destreg(as, ir, RID_RET); /* GCcdata * */ | ||
951 | |||
952 | /* Initialize immutable cdata object. */ | ||
953 | if (ir->o == IR_CNEWI) { | ||
954 | int32_t ofs = sizeof(GCcdata); | ||
955 | lua_assert(sz == 4 || sz == 8); | ||
956 | if (sz == 8) { | ||
957 | ofs += 4; ir++; | ||
958 | lua_assert(ir->o == IR_HIOP); | ||
959 | } | ||
960 | for (;;) { | ||
961 | Reg r = ra_alloc1(as, ir->op2, allow); | ||
962 | emit_lso(as, ARMI_STR, r, RID_RET, ofs); | ||
963 | rset_clear(allow, r); | ||
964 | if (ofs == sizeof(GCcdata)) break; | ||
965 | ofs -= 4; ir--; | ||
966 | } | ||
967 | } | ||
968 | /* Initialize gct and typeid. lj_mem_newgco() already sets marked. */ | ||
969 | { | ||
970 | uint32_t k = emit_isk12(ARMI_MOV, typeid); | ||
971 | Reg r = k ? RID_R1 : ra_allock(as, typeid, allow); | ||
972 | emit_lso(as, ARMI_STRB, RID_TMP, RID_RET, offsetof(GCcdata, gct)); | ||
973 | emit_lsox(as, ARMI_STRH, r, RID_RET, offsetof(GCcdata, typeid)); | ||
974 | emit_d(as, ARMI_MOV|ARMI_K12|~LJ_TCDATA, RID_TMP); | ||
975 | if (k) emit_d(as, ARMI_MOV^k, RID_R1); | ||
976 | } | ||
977 | asm_gencall(as, ci, args); | ||
978 | ra_allockreg(as, (int32_t)(sz+sizeof(GCcdata)), | ||
979 | ra_releasetmp(as, ASMREF_TMP1)); | ||
980 | } | ||
981 | #else | ||
982 | #define asm_cnew(as, ir) ((void)0) | ||
983 | #endif | ||
984 | |||
985 | /* -- Write barriers ------------------------------------------------------ */ | ||
986 | |||
987 | static void asm_tbar(ASMState *as, IRIns *ir) | ||
988 | { | ||
989 | Reg tab = ra_alloc1(as, ir->op1, RSET_GPR); | ||
990 | Reg link = ra_scratch(as, rset_exclude(RSET_GPR, tab)); | ||
991 | Reg gr = ra_allock(as, i32ptr(J2G(as->J)), | ||
992 | rset_exclude(rset_exclude(RSET_GPR, tab), link)); | ||
993 | Reg mark = RID_TMP; | ||
994 | MCLabel l_end = emit_label(as); | ||
995 | emit_lso(as, ARMI_STR, link, tab, (int32_t)offsetof(GCtab, gclist)); | ||
996 | emit_lso(as, ARMI_STRB, mark, tab, (int32_t)offsetof(GCtab, marked)); | ||
997 | emit_lso(as, ARMI_STR, tab, gr, | ||
998 | (int32_t)offsetof(global_State, gc.grayagain)); | ||
999 | emit_dn(as, ARMI_BIC|ARMI_K12|LJ_GC_BLACK, mark, mark); | ||
1000 | emit_lso(as, ARMI_LDR, link, gr, | ||
1001 | (int32_t)offsetof(global_State, gc.grayagain)); | ||
1002 | emit_branch(as, ARMF_CC(ARMI_B, CC_EQ), l_end); | ||
1003 | emit_n(as, ARMI_TST|ARMI_K12|LJ_GC_BLACK, mark); | ||
1004 | emit_lso(as, ARMI_LDRB, mark, tab, (int32_t)offsetof(GCtab, marked)); | ||
1005 | } | ||
1006 | |||
1007 | static void asm_obar(ASMState *as, IRIns *ir) | ||
1008 | { | ||
1009 | const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_gc_barrieruv]; | ||
1010 | IRRef args[2]; | ||
1011 | MCLabel l_end; | ||
1012 | Reg obj, val, tmp; | ||
1013 | /* No need for other object barriers (yet). */ | ||
1014 | lua_assert(IR(ir->op1)->o == IR_UREFC); | ||
1015 | ra_evictset(as, RSET_SCRATCH); | ||
1016 | l_end = emit_label(as); | ||
1017 | args[0] = ASMREF_TMP1; /* global_State *g */ | ||
1018 | args[1] = ir->op1; /* TValue *tv */ | ||
1019 | asm_gencall(as, ci, args); | ||
1020 | if ((*as->mcp >> 28) == CC_AL) | ||
1021 | *as->mcp = ARMF_CC(*as->mcp, CC_NE); | ||
1022 | else | ||
1023 | emit_branch(as, ARMF_CC(ARMI_B, CC_EQ), l_end); | ||
1024 | ra_allockreg(as, i32ptr(J2G(as->J)), ra_releasetmp(as, ASMREF_TMP1)); | ||
1025 | obj = IR(ir->op1)->r; | ||
1026 | tmp = ra_scratch(as, rset_exclude(RSET_GPR, obj)); | ||
1027 | emit_n(as, ARMF_CC(ARMI_TST, CC_NE)|ARMI_K12|LJ_GC_BLACK, tmp); | ||
1028 | emit_n(as, ARMI_TST|ARMI_K12|LJ_GC_WHITES, RID_TMP); | ||
1029 | val = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, obj)); | ||
1030 | emit_lso(as, ARMI_LDRB, tmp, obj, | ||
1031 | (int32_t)offsetof(GCupval, marked)-(int32_t)offsetof(GCupval, tv)); | ||
1032 | emit_lso(as, ARMI_LDRB, RID_TMP, val, (int32_t)offsetof(GChead, marked)); | ||
1033 | } | ||
1034 | |||
1035 | /* -- Arithmetic and logic operations ------------------------------------- */ | ||
1036 | |||
1037 | static int asm_swapops(ASMState *as, IRRef lref, IRRef rref) | ||
1038 | { | ||
1039 | IRIns *ir; | ||
1040 | if (irref_isk(rref)) | ||
1041 | return 0; /* Don't swap constants to the left. */ | ||
1042 | if (irref_isk(lref)) | ||
1043 | return 1; /* But swap constants to the right. */ | ||
1044 | ir = IR(rref); | ||
1045 | if ((ir->o >= IR_BSHL && ir->o <= IR_BROR) || | ||
1046 | (ir->o == IR_ADD && ir->op1 == ir->op2)) | ||
1047 | return 0; /* Don't swap fusable operands to the left. */ | ||
1048 | ir = IR(lref); | ||
1049 | if ((ir->o >= IR_BSHL && ir->o <= IR_BROR) || | ||
1050 | (ir->o == IR_ADD && ir->op1 == ir->op2)) | ||
1051 | return 1; /* But swap fusable operands to the right. */ | ||
1052 | return 0; /* Otherwise don't swap. */ | ||
1053 | } | ||
1054 | |||
1055 | static void asm_intop(ASMState *as, IRIns *ir, ARMIns ai) | ||
1056 | { | ||
1057 | IRRef lref = ir->op1, rref = ir->op2; | ||
1058 | Reg left, dest = ra_dest(as, ir, RSET_GPR); | ||
1059 | uint32_t m; | ||
1060 | if (asm_swapops(as, lref, rref)) { | ||
1061 | IRRef tmp = lref; lref = rref; rref = tmp; | ||
1062 | if ((ai & ~ARMI_S) == ARMI_SUB || (ai & ~ARMI_S) == ARMI_SBC) | ||
1063 | ai ^= (ARMI_SUB^ARMI_RSB); | ||
1064 | } | ||
1065 | left = ra_hintalloc(as, lref, dest, RSET_GPR); | ||
1066 | m = asm_fuseopm(as, ai, rref, rset_exclude(RSET_GPR, left)); | ||
1067 | if (irt_isguard(ir->t)) { /* For IR_ADDOV etc. */ | ||
1068 | asm_guardcc(as, CC_VS); | ||
1069 | ai |= ARMI_S; | ||
1070 | } | ||
1071 | emit_dn(as, ai^m, dest, left); | ||
1072 | } | ||
1073 | |||
1074 | static void asm_bitop(ASMState *as, IRIns *ir, ARMIns ai) | ||
1075 | { | ||
1076 | if (as->flagmcp == as->mcp) { /* Try to drop cmp r, #0. */ | ||
1077 | uint32_t cc = (as->mcp[1] >> 28); | ||
1078 | as->flagmcp = NULL; | ||
1079 | if (cc <= CC_NE) { | ||
1080 | as->mcp++; | ||
1081 | ai |= ARMI_S; | ||
1082 | } else if (cc == CC_GE) { | ||
1083 | *++as->mcp ^= ((CC_GE^CC_PL) << 28); | ||
1084 | ai |= ARMI_S; | ||
1085 | } else if (cc == CC_LT) { | ||
1086 | *++as->mcp ^= ((CC_LT^CC_MI) << 28); | ||
1087 | ai |= ARMI_S; | ||
1088 | } /* else: other conds don't work with bit ops. */ | ||
1089 | } | ||
1090 | if (ir->op2 == 0) { | ||
1091 | Reg dest = ra_dest(as, ir, RSET_GPR); | ||
1092 | uint32_t m = asm_fuseopm(as, ai, ir->op1, RSET_GPR); | ||
1093 | emit_d(as, ai^m, dest); | ||
1094 | } else { | ||
1095 | /* NYI: Turn BAND !k12 into uxtb, uxth or bfc or shl+shr. */ | ||
1096 | asm_intop(as, ir, ai); | ||
1097 | } | ||
1098 | } | ||
1099 | |||
1100 | static void asm_arithop(ASMState *as, IRIns *ir, ARMIns ai) | ||
1101 | { | ||
1102 | if (as->flagmcp == as->mcp) { /* Drop cmp r, #0. */ | ||
1103 | as->flagmcp = NULL; | ||
1104 | as->mcp++; | ||
1105 | ai |= ARMI_S; | ||
1106 | } | ||
1107 | asm_intop(as, ir, ai); | ||
1108 | } | ||
1109 | |||
1110 | static void asm_intneg(ASMState *as, IRIns *ir, ARMIns ai) | ||
1111 | { | ||
1112 | Reg dest = ra_dest(as, ir, RSET_GPR); | ||
1113 | Reg left = ra_hintalloc(as, ir->op1, dest, RSET_GPR); | ||
1114 | emit_dn(as, ai|ARMI_K12|0, dest, left); | ||
1115 | } | ||
1116 | |||
1117 | /* NYI: use add/shift for MUL(OV) with constants. FOLD only does 2^k. */ | ||
1118 | static void asm_intmul(ASMState *as, IRIns *ir) | ||
1119 | { | ||
1120 | Reg dest = ra_dest(as, ir, RSET_GPR); | ||
1121 | Reg left = ra_alloc1(as, ir->op1, rset_exclude(RSET_GPR, dest)); | ||
1122 | Reg right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left)); | ||
1123 | Reg tmp = RID_NONE; | ||
1124 | /* ARMv5 restriction: dest != left and dest_hi != left. */ | ||
1125 | if (dest == left && left != right) { left = right; right = dest; } | ||
1126 | if (irt_isguard(ir->t)) { /* IR_MULOV */ | ||
1127 | if (!(as->flags & JIT_F_ARMV6) && dest == left) | ||
1128 | tmp = left = ra_scratch(as, rset_exclude(RSET_FPR, left)); | ||
1129 | asm_guardcc(as, CC_NE); | ||
1130 | emit_nm(as, ARMI_TEQ|ARMF_SH(ARMSH_ASR, 31), RID_TMP, dest); | ||
1131 | emit_dnm(as, ARMI_SMULL|ARMF_S(right), dest, RID_TMP, left); | ||
1132 | } else { | ||
1133 | if (!(as->flags & JIT_F_ARMV6) && dest == left) tmp = left = RID_TMP; | ||
1134 | emit_nm(as, ARMI_MUL|ARMF_S(right), dest, left); | ||
1135 | } | ||
1136 | /* Only need this for the dest == left == right case. */ | ||
1137 | if (ra_hasreg(tmp)) emit_dm(as, ARMI_MOV, tmp, right); | ||
1138 | } | ||
1139 | |||
1140 | static void asm_intmod(ASMState *as, IRIns *ir) | ||
1141 | { | ||
1142 | const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_vm_modi]; | ||
1143 | IRRef args[2]; | ||
1144 | args[0] = ir->op1; | ||
1145 | args[1] = ir->op2; | ||
1146 | asm_setupresult(as, ir, ci); | ||
1147 | asm_gencall(as, ci, args); | ||
1148 | } | ||
1149 | |||
1150 | static void asm_bitswap(ASMState *as, IRIns *ir) | ||
1151 | { | ||
1152 | Reg dest = ra_dest(as, ir, RSET_GPR); | ||
1153 | Reg left = ra_alloc1(as, ir->op1, RSET_GPR); | ||
1154 | if ((as->flags & JIT_F_ARMV6)) { | ||
1155 | emit_dm(as, ARMI_REV, dest, left); | ||
1156 | } else { | ||
1157 | Reg tmp2 = dest; | ||
1158 | if (tmp2 == left) | ||
1159 | tmp2 = ra_scratch(as, rset_exclude(rset_exclude(RSET_GPR, dest), left)); | ||
1160 | emit_dnm(as, ARMI_EOR|ARMF_SH(ARMSH_LSR, 8), dest, tmp2, RID_TMP); | ||
1161 | emit_dm(as, ARMI_MOV|ARMF_SH(ARMSH_ROR, 8), tmp2, left); | ||
1162 | emit_dn(as, ARMI_BIC|ARMI_K12|256*8|255, RID_TMP, RID_TMP); | ||
1163 | emit_dnm(as, ARMI_EOR|ARMF_SH(ARMSH_ROR, 16), RID_TMP, left, left); | ||
1164 | } | ||
1165 | } | ||
1166 | |||
1167 | static void asm_bitshift(ASMState *as, IRIns *ir, ARMShift sh) | ||
1168 | { | ||
1169 | if (irref_isk(ir->op2)) { /* Constant shifts. */ | ||
1170 | /* NYI: Turn SHL+SHR or BAND+SHR into uxtb, uxth or ubfx. */ | ||
1171 | /* NYI: Turn SHL+ASR into sxtb, sxth or sbfx. */ | ||
1172 | Reg dest = ra_dest(as, ir, RSET_GPR); | ||
1173 | Reg left = ra_alloc1(as, ir->op1, RSET_GPR); | ||
1174 | int32_t shift = (IR(ir->op2)->i & 31); | ||
1175 | emit_dm(as, ARMI_MOV|ARMF_SH(sh, shift), dest, left); | ||
1176 | } else { | ||
1177 | Reg dest = ra_dest(as, ir, RSET_GPR); | ||
1178 | Reg left = ra_alloc1(as, ir->op1, RSET_GPR); | ||
1179 | Reg right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left)); | ||
1180 | emit_dm(as, ARMI_MOV|ARMF_RSH(sh, right), dest, left); | ||
1181 | } | ||
1182 | } | ||
1183 | |||
1184 | static void asm_intmin_max(ASMState *as, IRIns *ir, int cc) | ||
1185 | { | ||
1186 | uint32_t kcmp = 0, kmov = 0; | ||
1187 | Reg dest = ra_dest(as, ir, RSET_GPR); | ||
1188 | Reg left = ra_hintalloc(as, ir->op1, dest, RSET_GPR); | ||
1189 | Reg right = 0; | ||
1190 | if (irref_isk(ir->op2)) { | ||
1191 | kcmp = emit_isk12(ARMI_CMP, IR(ir->op2)->i); | ||
1192 | if (kcmp) kmov = emit_isk12(ARMI_MOV, IR(ir->op2)->i); | ||
1193 | } | ||
1194 | if (!kmov) { | ||
1195 | kcmp = 0; | ||
1196 | right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left)); | ||
1197 | } | ||
1198 | if (dest != right) { | ||
1199 | emit_dm(as, ARMF_CC(ARMI_MOV, cc)^kmov, dest, right); | ||
1200 | cc ^= 1; /* Must use opposite conditions for paired moves. */ | ||
1201 | } else { | ||
1202 | cc ^= (CC_LT^CC_GT); /* Otherwise may swap CC_LT <-> CC_GT. */ | ||
1203 | } | ||
1204 | if (dest != left) emit_dm(as, ARMF_CC(ARMI_MOV, cc)^kmov, dest, left); | ||
1205 | emit_nm(as, ARMI_CMP^kcmp, left, right); | ||
1206 | } | ||
1207 | |||
1208 | static void asm_fpmin_max(ASMState *as, IRIns *ir, int cc) | ||
1209 | { | ||
1210 | const CCallInfo *ci = &lj_ir_callinfo[IRCALL_softfp_cmp]; | ||
1211 | RegSet drop = RSET_SCRATCH; | ||
1212 | Reg r; | ||
1213 | IRRef args[4]; | ||
1214 | args[0] = ir->op1; args[1] = (ir+1)->op1; | ||
1215 | args[2] = ir->op2; args[3] = (ir+1)->op2; | ||
1216 | /* __aeabi_cdcmple preserves r0-r3. */ | ||
1217 | if (ra_hasreg(ir->r)) rset_clear(drop, ir->r); | ||
1218 | if (ra_hasreg((ir+1)->r)) rset_clear(drop, (ir+1)->r); | ||
1219 | if (!rset_test(as->freeset, RID_R2) && | ||
1220 | regcost_ref(as->cost[RID_R2]) == args[2]) rset_clear(drop, RID_R2); | ||
1221 | if (!rset_test(as->freeset, RID_R3) && | ||
1222 | regcost_ref(as->cost[RID_R3]) == args[3]) rset_clear(drop, RID_R3); | ||
1223 | ra_evictset(as, drop); | ||
1224 | ra_destpair(as, ir); | ||
1225 | emit_dm(as, ARMF_CC(ARMI_MOV, cc), RID_RETHI, RID_R3); | ||
1226 | emit_dm(as, ARMF_CC(ARMI_MOV, cc), RID_RETLO, RID_R2); | ||
1227 | emit_call(as, (void *)ci->func); | ||
1228 | for (r = RID_R0; r <= RID_R3; r++) | ||
1229 | ra_leftov(as, r, args[r-RID_R0]); | ||
1230 | } | ||
1231 | |||
1232 | /* -- Comparisons --------------------------------------------------------- */ | ||
1233 | |||
1234 | /* Map of comparisons to flags. ORDER IR. */ | ||
1235 | static const uint8_t asm_compmap[IR_ABC+1] = { | ||
1236 | /* op FP swp int cc FP cc */ | ||
1237 | /* LT */ CC_GE + (CC_HS << 4), | ||
1238 | /* GE x */ CC_LT + (CC_HI << 4), | ||
1239 | /* LE */ CC_GT + (CC_HI << 4), | ||
1240 | /* GT x */ CC_LE + (CC_HS << 4), | ||
1241 | /* ULT x */ CC_HS + (CC_LS << 4), | ||
1242 | /* UGE */ CC_LO + (CC_LO << 4), | ||
1243 | /* ULE x */ CC_HI + (CC_LO << 4), | ||
1244 | /* UGT */ CC_LS + (CC_LS << 4), | ||
1245 | /* EQ */ CC_NE + (CC_NE << 4), | ||
1246 | /* NE */ CC_EQ + (CC_EQ << 4), | ||
1247 | /* ABC */ CC_LS + (CC_LS << 4) /* Same as UGT. */ | ||
1248 | }; | ||
1249 | |||
1250 | /* FP comparisons. */ | ||
1251 | static void asm_fpcomp(ASMState *as, IRIns *ir) | ||
1252 | { | ||
1253 | const CCallInfo *ci = &lj_ir_callinfo[IRCALL_softfp_cmp]; | ||
1254 | RegSet drop = RSET_SCRATCH; | ||
1255 | Reg r; | ||
1256 | IRRef args[4]; | ||
1257 | int swp = (((ir->o ^ (ir->o >> 2)) & ~(ir->o >> 3) & 1) << 1); | ||
1258 | args[swp^0] = ir->op1; args[swp^1] = (ir+1)->op1; | ||
1259 | args[swp^2] = ir->op2; args[swp^3] = (ir+1)->op2; | ||
1260 | /* __aeabi_cdcmple preserves r0-r3. This helps to reduce spills. */ | ||
1261 | for (r = RID_R0; r <= RID_R3; r++) | ||
1262 | if (!rset_test(as->freeset, r) && | ||
1263 | regcost_ref(as->cost[r]) == args[r-RID_R0]) rset_clear(drop, r); | ||
1264 | ra_evictset(as, drop); | ||
1265 | asm_guardcc(as, (asm_compmap[ir->o] >> 4)); | ||
1266 | emit_call(as, (void *)ci->func); | ||
1267 | for (r = RID_R0; r <= RID_R3; r++) | ||
1268 | ra_leftov(as, r, args[r-RID_R0]); | ||
1269 | } | ||
1270 | |||
1271 | /* Integer comparisons. */ | ||
1272 | static void asm_intcomp(ASMState *as, IRIns *ir) | ||
1273 | { | ||
1274 | ARMCC cc = (asm_compmap[ir->o] & 15); | ||
1275 | IRRef lref = ir->op1, rref = ir->op2; | ||
1276 | Reg left; | ||
1277 | uint32_t m; | ||
1278 | int cmpprev0 = 0; | ||
1279 | lua_assert(irt_isint(ir->t) || irt_isaddr(ir->t)); | ||
1280 | if (asm_swapops(as, lref, rref)) { | ||
1281 | Reg tmp = lref; lref = rref; rref = tmp; | ||
1282 | if (cc >= CC_GE) cc ^= 7; /* LT <-> GT, LE <-> GE */ | ||
1283 | else if (cc > CC_NE) cc ^= 11; /* LO <-> HI, LS <-> HS */ | ||
1284 | } | ||
1285 | if (irref_isk(rref) && IR(rref)->i == 0) { | ||
1286 | IRIns *irl = IR(lref); | ||
1287 | cmpprev0 = (irl+1 == ir); | ||
1288 | /* Combine comp(BAND(left, right), 0) into tst left, right. */ | ||
1289 | if (cmpprev0 && irl->o == IR_BAND && !ra_used(irl)) { | ||
1290 | IRRef blref = irl->op1, brref = irl->op2; | ||
1291 | uint32_t m2 = 0; | ||
1292 | Reg bleft; | ||
1293 | if (asm_swapops(as, blref, brref)) { | ||
1294 | Reg tmp = blref; blref = brref; brref = tmp; | ||
1295 | } | ||
1296 | if (irref_isk(brref)) { | ||
1297 | m2 = emit_isk12(ARMI_AND, IR(brref)->i); | ||
1298 | if ((m2 & (ARMI_AND^ARMI_BIC))) | ||
1299 | goto notst; /* Not beneficial if we miss a constant operand. */ | ||
1300 | } | ||
1301 | if (cc == CC_GE) cc = CC_PL; | ||
1302 | else if (cc == CC_LT) cc = CC_MI; | ||
1303 | else if (cc > CC_NE) goto notst; /* Other conds don't work with tst. */ | ||
1304 | bleft = ra_alloc1(as, blref, RSET_GPR); | ||
1305 | if (!m2) m2 = asm_fuseopm(as, 0, brref, rset_exclude(RSET_GPR, bleft)); | ||
1306 | asm_guardcc(as, cc); | ||
1307 | emit_n(as, ARMI_TST^m2, bleft); | ||
1308 | return; | ||
1309 | } | ||
1310 | } | ||
1311 | notst: | ||
1312 | left = ra_alloc1(as, lref, RSET_GPR); | ||
1313 | m = asm_fuseopm(as, ARMI_CMP, rref, rset_exclude(RSET_GPR, left)); | ||
1314 | asm_guardcc(as, cc); | ||
1315 | emit_n(as, ARMI_CMP^m, left); | ||
1316 | /* Signed comparison with zero and referencing previous ins? */ | ||
1317 | if (cmpprev0 && (cc <= CC_NE || cc >= CC_GE)) | ||
1318 | as->flagmcp = as->mcp; /* Allow elimination of the compare. */ | ||
1319 | } | ||
1320 | |||
1321 | /* 64 bit integer comparisons. */ | ||
1322 | static void asm_int64comp(ASMState *as, IRIns *ir) | ||
1323 | { | ||
1324 | int signedcomp = (ir->o <= IR_GT); | ||
1325 | ARMCC cclo, cchi; | ||
1326 | Reg leftlo, lefthi; | ||
1327 | uint32_t mlo, mhi; | ||
1328 | RegSet allow = RSET_GPR, oldfree; | ||
1329 | |||
1330 | /* Always use unsigned comparison for loword. */ | ||
1331 | cclo = asm_compmap[ir->o + (signedcomp ? 4 : 0)] & 15; | ||
1332 | leftlo = ra_alloc1(as, ir->op1, allow); | ||
1333 | oldfree = as->freeset; | ||
1334 | mlo = asm_fuseopm(as, ARMI_CMP, ir->op2, rset_clear(allow, leftlo)); | ||
1335 | allow &= ~(oldfree & ~as->freeset); /* Update for allocs of asm_fuseopm. */ | ||
1336 | |||
1337 | /* Use signed or unsigned comparison for hiword. */ | ||
1338 | cchi = asm_compmap[ir->o] & 15; | ||
1339 | lefthi = ra_alloc1(as, (ir+1)->op1, allow); | ||
1340 | mhi = asm_fuseopm(as, ARMI_CMP, (ir+1)->op2, rset_clear(allow, lefthi)); | ||
1341 | |||
1342 | /* All register allocations must be performed _before_ this point. */ | ||
1343 | if (signedcomp) { | ||
1344 | MCLabel l_around = emit_label(as); | ||
1345 | asm_guardcc(as, cclo); | ||
1346 | emit_n(as, ARMI_CMP^mlo, leftlo); | ||
1347 | emit_branch(as, ARMF_CC(ARMI_B, CC_NE), l_around); | ||
1348 | if (cchi == CC_GE || cchi == CC_LE) cchi ^= 6; /* GE -> GT, LE -> LT */ | ||
1349 | asm_guardcc(as, cchi); | ||
1350 | } else { | ||
1351 | asm_guardcc(as, cclo); | ||
1352 | emit_n(as, ARMF_CC(ARMI_CMP, CC_EQ)^mlo, leftlo); | ||
1353 | } | ||
1354 | emit_n(as, ARMI_CMP^mhi, lefthi); | ||
1355 | } | ||
1356 | |||
1357 | /* -- Support for 64 bit ops in 32 bit mode ------------------------------- */ | ||
1358 | |||
1359 | /* Hiword op of a split 64 bit op. Previous op must be the loword op. */ | ||
1360 | static void asm_hiop(ASMState *as, IRIns *ir) | ||
1361 | { | ||
1362 | /* HIOP is marked as a store because it needs its own DCE logic. */ | ||
1363 | int uselo = ra_used(ir-1), usehi = ra_used(ir); /* Loword/hiword used? */ | ||
1364 | if (LJ_UNLIKELY(!(as->flags & JIT_F_OPT_DCE))) uselo = usehi = 1; | ||
1365 | if ((ir-1)->o <= IR_NE) { /* 64 bit integer or FP comparisons. ORDER IR. */ | ||
1366 | as->curins--; /* Always skip the loword comparison. */ | ||
1367 | if (irt_isint(ir->t)) | ||
1368 | asm_int64comp(as, ir-1); | ||
1369 | else | ||
1370 | asm_fpcomp(as, ir-1); | ||
1371 | return; | ||
1372 | } else if ((ir-1)->o == IR_MIN || (ir-1)->o == IR_MAX) { | ||
1373 | as->curins--; /* Always skip the loword min/max. */ | ||
1374 | if (uselo || usehi) | ||
1375 | asm_fpmin_max(as, ir-1, (ir-1)->o == IR_MIN ? CC_HI : CC_LO); | ||
1376 | return; | ||
1377 | } | ||
1378 | if (!usehi) return; /* Skip unused hiword op for all remaining ops. */ | ||
1379 | switch ((ir-1)->o) { | ||
1380 | #if LJ_HASFFI | ||
1381 | case IR_ADD: | ||
1382 | as->curins--; | ||
1383 | asm_intop(as, ir, ARMI_ADC); | ||
1384 | asm_intop(as, ir-1, ARMI_ADD|ARMI_S); | ||
1385 | break; | ||
1386 | case IR_SUB: | ||
1387 | as->curins--; | ||
1388 | asm_intop(as, ir, ARMI_SBC); | ||
1389 | asm_intop(as, ir-1, ARMI_SUB|ARMI_S); | ||
1390 | break; | ||
1391 | case IR_NEG: | ||
1392 | as->curins--; | ||
1393 | asm_intneg(as, ir, ARMI_RSC); | ||
1394 | asm_intneg(as, ir-1, ARMI_RSB|ARMI_S); | ||
1395 | break; | ||
1396 | #endif | ||
1397 | case IR_SLOAD: case IR_ALOAD: case IR_HLOAD: case IR_ULOAD: case IR_VLOAD: | ||
1398 | case IR_STRTO: | ||
1399 | if (!uselo) | ||
1400 | ra_allocref(as, ir->op1, RSET_GPR); /* Mark lo op as used. */ | ||
1401 | break; | ||
1402 | case IR_CALLN: | ||
1403 | case IR_CALLS: | ||
1404 | case IR_CALLXS: | ||
1405 | if (!uselo) | ||
1406 | ra_allocref(as, ir->op1, RID2RSET(RID_RETLO)); /* Mark lo op as used. */ | ||
1407 | break; | ||
1408 | case IR_ASTORE: case IR_HSTORE: case IR_USTORE: | ||
1409 | case IR_TOSTR: case IR_CNEWI: | ||
1410 | /* Nothing to do here. Handled by lo op itself. */ | ||
1411 | break; | ||
1412 | default: lua_assert(0); break; | ||
1413 | } | ||
1414 | } | ||
1415 | |||
1416 | /* -- Stack handling ------------------------------------------------------ */ | ||
1417 | |||
1418 | /* Check Lua stack size for overflow. Use exit handler as fallback. */ | ||
1419 | static void asm_stack_check(ASMState *as, BCReg topslot, | ||
1420 | IRIns *irp, RegSet allow, ExitNo exitno) | ||
1421 | { | ||
1422 | Reg pbase; | ||
1423 | uint32_t k; | ||
1424 | if (irp) { | ||
1425 | if (!ra_hasspill(irp->s)) { | ||
1426 | pbase = irp->r; | ||
1427 | lua_assert(ra_hasreg(pbase)); | ||
1428 | } else if (allow) { | ||
1429 | pbase = rset_pickbot(allow); | ||
1430 | } else { | ||
1431 | pbase = RID_RET; | ||
1432 | emit_lso(as, ARMI_LDR, RID_RET, RID_SP, 0); /* Restore temp. register. */ | ||
1433 | } | ||
1434 | } else { | ||
1435 | pbase = RID_BASE; | ||
1436 | } | ||
1437 | emit_branch(as, ARMF_CC(ARMI_BL, CC_LS), exitstub_addr(as->J, exitno)); | ||
1438 | k = emit_isk12(0, (int32_t)(8*topslot)); | ||
1439 | lua_assert(k); | ||
1440 | emit_n(as, ARMI_CMP^k, RID_TMP); | ||
1441 | emit_dnm(as, ARMI_SUB, RID_TMP, RID_TMP, pbase); | ||
1442 | emit_lso(as, ARMI_LDR, RID_TMP, RID_TMP, | ||
1443 | (int32_t)offsetof(lua_State, maxstack)); | ||
1444 | if (irp) { /* Must not spill arbitrary registers in head of side trace. */ | ||
1445 | int32_t i = i32ptr(&J2G(as->J)->jit_L); | ||
1446 | if (ra_hasspill(irp->s)) | ||
1447 | emit_lso(as, ARMI_LDR, pbase, RID_SP, sps_scale(irp->s)); | ||
1448 | emit_lso(as, ARMI_LDR, RID_TMP, RID_TMP, (i & 4095)); | ||
1449 | if (ra_hasspill(irp->s) && !allow) | ||
1450 | emit_lso(as, ARMI_STR, RID_RET, RID_SP, 0); /* Save temp. register. */ | ||
1451 | emit_loadi(as, RID_TMP, (i & ~4095)); | ||
1452 | } else { | ||
1453 | emit_getgl(as, RID_TMP, jit_L); | ||
1454 | } | ||
1455 | } | ||
1456 | |||
1457 | /* Restore Lua stack from on-trace state. */ | ||
1458 | static void asm_stack_restore(ASMState *as, SnapShot *snap) | ||
1459 | { | ||
1460 | SnapEntry *map = &as->T->snapmap[snap->mapofs]; | ||
1461 | SnapEntry *flinks = &as->T->snapmap[snap_nextofs(as->T, snap)-1]; | ||
1462 | MSize n, nent = snap->nent; | ||
1463 | /* Store the value of all modified slots to the Lua stack. */ | ||
1464 | for (n = 0; n < nent; n++) { | ||
1465 | SnapEntry sn = map[n]; | ||
1466 | BCReg s = snap_slot(sn); | ||
1467 | int32_t ofs = 8*((int32_t)s-1); | ||
1468 | IRRef ref = snap_ref(sn); | ||
1469 | IRIns *ir = IR(ref); | ||
1470 | if ((sn & SNAP_NORESTORE)) | ||
1471 | continue; | ||
1472 | if (irt_isnum(ir->t)) { | ||
1473 | RegSet odd = rset_exclude(RSET_GPRODD, RID_BASE); | ||
1474 | Reg tmp; | ||
1475 | lua_assert(irref_isk(ref)); /* LJ_SOFTFP: must be a number constant. */ | ||
1476 | tmp = ra_allock(as, (int32_t)ir_knum(ir)->u32.lo, | ||
1477 | rset_exclude(RSET_GPREVEN, RID_BASE)); | ||
1478 | emit_lso(as, ARMI_STR, tmp, RID_BASE, ofs); | ||
1479 | if (rset_test(as->freeset, tmp+1)) odd = RID2RSET(tmp+1); | ||
1480 | tmp = ra_allock(as, (int32_t)ir_knum(ir)->u32.hi, odd); | ||
1481 | emit_lso(as, ARMI_STR, tmp, RID_BASE, ofs+4); | ||
1482 | } else { | ||
1483 | RegSet odd = rset_exclude(RSET_GPRODD, RID_BASE); | ||
1484 | Reg type; | ||
1485 | lua_assert(irt_ispri(ir->t) || irt_isaddr(ir->t) || irt_isinteger(ir->t)); | ||
1486 | if (!irt_ispri(ir->t)) { | ||
1487 | Reg src = ra_alloc1(as, ref, rset_exclude(RSET_GPREVEN, RID_BASE)); | ||
1488 | emit_lso(as, ARMI_STR, src, RID_BASE, ofs); | ||
1489 | if (rset_test(as->freeset, src+1)) odd = RID2RSET(src+1); | ||
1490 | } | ||
1491 | if ((sn & (SNAP_CONT|SNAP_FRAME))) { | ||
1492 | if (s == 0) continue; /* Do not overwrite link to previous frame. */ | ||
1493 | type = ra_allock(as, (int32_t)(*flinks--), odd); | ||
1494 | } else if ((sn & SNAP_SOFTFPNUM)) { | ||
1495 | type = ra_alloc1(as, ref+1, rset_exclude(RSET_GPRODD, RID_BASE)); | ||
1496 | } else { | ||
1497 | type = ra_allock(as, (int32_t)irt_toitype(ir->t), odd); | ||
1498 | } | ||
1499 | emit_lso(as, ARMI_STR, type, RID_BASE, ofs+4); | ||
1500 | } | ||
1501 | checkmclim(as); | ||
1502 | } | ||
1503 | lua_assert(map + nent == flinks); | ||
1504 | } | ||
1505 | |||
1506 | /* -- GC handling --------------------------------------------------------- */ | ||
1507 | |||
1508 | /* Check GC threshold and do one or more GC steps. */ | ||
1509 | static void asm_gc_check(ASMState *as) | ||
1510 | { | ||
1511 | const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_gc_step_jit]; | ||
1512 | IRRef args[2]; | ||
1513 | MCLabel l_end; | ||
1514 | Reg tmp1, tmp2; | ||
1515 | ra_evictset(as, RSET_SCRATCH); | ||
1516 | l_end = emit_label(as); | ||
1517 | /* Exit trace if in GCSatomic or GCSfinalize. Avoids syncing GC objects. */ | ||
1518 | asm_guardcc(as, CC_NE); /* Assumes asm_snap_prep() already done. */ | ||
1519 | emit_n(as, ARMI_CMP|ARMI_K12|0, RID_RET); | ||
1520 | args[0] = ASMREF_TMP1; /* global_State *g */ | ||
1521 | args[1] = ASMREF_TMP2; /* MSize steps */ | ||
1522 | asm_gencall(as, ci, args); | ||
1523 | tmp1 = ra_releasetmp(as, ASMREF_TMP1); | ||
1524 | tmp2 = ra_releasetmp(as, ASMREF_TMP2); | ||
1525 | emit_loadi(as, tmp2, (int32_t)as->gcsteps); | ||
1526 | /* Jump around GC step if GC total < GC threshold. */ | ||
1527 | emit_branch(as, ARMF_CC(ARMI_B, CC_LS), l_end); | ||
1528 | emit_nm(as, ARMI_CMP, RID_TMP, tmp2); | ||
1529 | emit_lso(as, ARMI_LDR, tmp2, tmp1, | ||
1530 | (int32_t)offsetof(global_State, gc.threshold)); | ||
1531 | emit_lso(as, ARMI_LDR, RID_TMP, tmp1, | ||
1532 | (int32_t)offsetof(global_State, gc.total)); | ||
1533 | ra_allockreg(as, i32ptr(J2G(as->J)), tmp1); | ||
1534 | as->gcsteps = 0; | ||
1535 | checkmclim(as); | ||
1536 | } | ||
1537 | |||
1538 | /* -- Loop handling ------------------------------------------------------- */ | ||
1539 | |||
1540 | /* Fixup the loop branch. */ | ||
1541 | static void asm_loop_fixup(ASMState *as) | ||
1542 | { | ||
1543 | MCode *p = as->mctop; | ||
1544 | MCode *target = as->mcp; | ||
1545 | if (as->loopinv) { /* Inverted loop branch? */ | ||
1546 | /* asm_guardcc already inverted the bcc and patched the final bl. */ | ||
1547 | p[-2] |= ((uint32_t)(target-p) & 0x00ffffffu); | ||
1548 | } else { | ||
1549 | p[-1] = ARMI_B | ((uint32_t)((target-p)-1) & 0x00ffffffu); | ||
1550 | } | ||
1551 | } | ||
1552 | |||
1553 | /* -- Head of trace ------------------------------------------------------- */ | ||
1554 | |||
1555 | /* Reload L register from g->jit_L. */ | ||
1556 | static void asm_head_lreg(ASMState *as) | ||
1557 | { | ||
1558 | IRIns *ir = IR(ASMREF_L); | ||
1559 | if (ra_used(ir)) { | ||
1560 | Reg r = ra_dest(as, ir, RSET_GPR); | ||
1561 | emit_getgl(as, r, jit_L); | ||
1562 | ra_evictk(as); | ||
1563 | } | ||
1564 | } | ||
1565 | |||
1566 | /* Coalesce BASE register for a root trace. */ | ||
1567 | static void asm_head_root_base(ASMState *as) | ||
1568 | { | ||
1569 | IRIns *ir; | ||
1570 | asm_head_lreg(as); | ||
1571 | ir = IR(REF_BASE); | ||
1572 | if (ra_hasreg(ir->r) && rset_test(as->modset, ir->r)) ra_spill(as, ir); | ||
1573 | ra_destreg(as, ir, RID_BASE); | ||
1574 | } | ||
1575 | |||
1576 | /* Coalesce BASE register for a side trace. */ | ||
1577 | static RegSet asm_head_side_base(ASMState *as, IRIns *irp, RegSet allow) | ||
1578 | { | ||
1579 | IRIns *ir; | ||
1580 | asm_head_lreg(as); | ||
1581 | ir = IR(REF_BASE); | ||
1582 | if (ra_hasreg(ir->r) && rset_test(as->modset, ir->r)) ra_spill(as, ir); | ||
1583 | if (ra_hasspill(irp->s)) { | ||
1584 | rset_clear(allow, ra_dest(as, ir, allow)); | ||
1585 | } else { | ||
1586 | lua_assert(ra_hasreg(irp->r)); | ||
1587 | rset_clear(allow, irp->r); | ||
1588 | ra_destreg(as, ir, irp->r); | ||
1589 | } | ||
1590 | return allow; | ||
1591 | } | ||
1592 | |||
1593 | /* -- Tail of trace ------------------------------------------------------- */ | ||
1594 | |||
1595 | /* Fixup the tail code. */ | ||
1596 | static void asm_tail_fixup(ASMState *as, TraceNo lnk) | ||
1597 | { | ||
1598 | MCode *p = as->mctop; | ||
1599 | MCode *target; | ||
1600 | int32_t spadj = as->T->spadjust; | ||
1601 | if (spadj == 0) { | ||
1602 | as->mctop = --p; | ||
1603 | } else { | ||
1604 | /* Patch stack adjustment. */ | ||
1605 | uint32_t k = emit_isk12(ARMI_ADD, spadj); | ||
1606 | lua_assert(k); | ||
1607 | p[-2] = (ARMI_ADD^k) | ARMF_D(RID_SP) | ARMF_N(RID_SP); | ||
1608 | } | ||
1609 | /* Patch exit branch. */ | ||
1610 | target = lnk ? traceref(as->J, lnk)->mcode : (MCode *)lj_vm_exit_interp; | ||
1611 | p[-1] = ARMI_B|(((target-p)-1)&0x00ffffffu); | ||
1612 | } | ||
1613 | |||
1614 | /* Prepare tail of code. */ | ||
1615 | static void asm_tail_prep(ASMState *as) | ||
1616 | { | ||
1617 | MCode *p = as->mctop - 1; /* Leave room for exit branch. */ | ||
1618 | if (as->loopref) { | ||
1619 | as->invmcp = as->mcp = p; | ||
1620 | } else { | ||
1621 | as->mcp = p-1; /* Leave room for stack pointer adjustment. */ | ||
1622 | as->invmcp = NULL; | ||
1623 | } | ||
1624 | *p = 0; /* Prevent load/store merging. */ | ||
1625 | } | ||
1626 | |||
1627 | /* -- Instruction dispatch ------------------------------------------------ */ | ||
1628 | |||
1629 | /* Assemble a single instruction. */ | ||
1630 | static void asm_ir(ASMState *as, IRIns *ir) | ||
1631 | { | ||
1632 | switch ((IROp)ir->o) { | ||
1633 | /* Miscellaneous ops. */ | ||
1634 | case IR_LOOP: asm_loop(as); break; | ||
1635 | case IR_NOP: case IR_XBAR: lua_assert(!ra_used(ir)); break; | ||
1636 | case IR_USE: ra_alloc1(as, ir->op1, RSET_GPR); break; | ||
1637 | case IR_PHI: asm_phi(as, ir); break; | ||
1638 | case IR_HIOP: asm_hiop(as, ir); break; | ||
1639 | |||
1640 | /* Guarded assertions. */ | ||
1641 | case IR_EQ: case IR_NE: | ||
1642 | if ((ir-1)->o == IR_HREF && ir->op1 == as->curins-1) { | ||
1643 | as->curins--; | ||
1644 | asm_href(as, ir-1, (IROp)ir->o); | ||
1645 | break; | ||
1646 | } | ||
1647 | /* fallthrough */ | ||
1648 | case IR_LT: case IR_GE: case IR_LE: case IR_GT: | ||
1649 | case IR_ULT: case IR_UGE: case IR_ULE: case IR_UGT: | ||
1650 | case IR_ABC: | ||
1651 | asm_intcomp(as, ir); | ||
1652 | break; | ||
1653 | |||
1654 | case IR_RETF: asm_retf(as, ir); break; | ||
1655 | |||
1656 | /* Bit ops. */ | ||
1657 | case IR_BNOT: asm_bitop(as, ir, ARMI_MVN); break; | ||
1658 | case IR_BSWAP: asm_bitswap(as, ir); break; | ||
1659 | |||
1660 | case IR_BAND: asm_bitop(as, ir, ARMI_AND); break; | ||
1661 | case IR_BOR: asm_bitop(as, ir, ARMI_ORR); break; | ||
1662 | case IR_BXOR: asm_bitop(as, ir, ARMI_EOR); break; | ||
1663 | |||
1664 | case IR_BSHL: asm_bitshift(as, ir, ARMSH_LSL); break; | ||
1665 | case IR_BSHR: asm_bitshift(as, ir, ARMSH_LSR); break; | ||
1666 | case IR_BSAR: asm_bitshift(as, ir, ARMSH_ASR); break; | ||
1667 | case IR_BROR: asm_bitshift(as, ir, ARMSH_ROR); break; | ||
1668 | case IR_BROL: lua_assert(0); break; | ||
1669 | |||
1670 | /* Arithmetic ops. */ | ||
1671 | case IR_ADD: case IR_ADDOV: asm_arithop(as, ir, ARMI_ADD); break; | ||
1672 | case IR_SUB: case IR_SUBOV: asm_arithop(as, ir, ARMI_SUB); break; | ||
1673 | case IR_MUL: case IR_MULOV: asm_intmul(as, ir); break; | ||
1674 | case IR_MOD: asm_intmod(as, ir); break; | ||
1675 | |||
1676 | case IR_NEG: asm_intneg(as, ir, ARMI_RSB); break; | ||
1677 | |||
1678 | case IR_MIN: asm_intmin_max(as, ir, CC_GT); break; | ||
1679 | case IR_MAX: asm_intmin_max(as, ir, CC_LT); break; | ||
1680 | |||
1681 | case IR_FPMATH: case IR_ATAN2: case IR_LDEXP: | ||
1682 | case IR_DIV: case IR_POW: case IR_ABS: case IR_TOBIT: | ||
1683 | lua_assert(0); /* Unused for LJ_SOFTFP. */ | ||
1684 | break; | ||
1685 | |||
1686 | /* Memory references. */ | ||
1687 | case IR_AREF: asm_aref(as, ir); break; | ||
1688 | case IR_HREF: asm_href(as, ir, 0); break; | ||
1689 | case IR_HREFK: asm_hrefk(as, ir); break; | ||
1690 | case IR_NEWREF: asm_newref(as, ir); break; | ||
1691 | case IR_UREFO: case IR_UREFC: asm_uref(as, ir); break; | ||
1692 | case IR_FREF: asm_fref(as, ir); break; | ||
1693 | case IR_STRREF: asm_strref(as, ir); break; | ||
1694 | |||
1695 | /* Loads and stores. */ | ||
1696 | case IR_ALOAD: case IR_HLOAD: case IR_ULOAD: case IR_VLOAD: | ||
1697 | asm_ahuvload(as, ir); | ||
1698 | break; | ||
1699 | case IR_FLOAD: asm_fload(as, ir); break; | ||
1700 | case IR_XLOAD: asm_xload(as, ir); break; | ||
1701 | case IR_SLOAD: asm_sload(as, ir); break; | ||
1702 | |||
1703 | case IR_ASTORE: case IR_HSTORE: case IR_USTORE: asm_ahustore(as, ir); break; | ||
1704 | case IR_FSTORE: asm_fstore(as, ir); break; | ||
1705 | case IR_XSTORE: asm_xstore(as, ir); break; | ||
1706 | |||
1707 | /* Allocations. */ | ||
1708 | case IR_SNEW: case IR_XSNEW: asm_snew(as, ir); break; | ||
1709 | case IR_TNEW: asm_tnew(as, ir); break; | ||
1710 | case IR_TDUP: asm_tdup(as, ir); break; | ||
1711 | case IR_CNEW: case IR_CNEWI: asm_cnew(as, ir); break; | ||
1712 | |||
1713 | /* Write barriers. */ | ||
1714 | case IR_TBAR: asm_tbar(as, ir); break; | ||
1715 | case IR_OBAR: asm_obar(as, ir); break; | ||
1716 | |||
1717 | /* Type conversions. */ | ||
1718 | case IR_CONV: asm_conv(as, ir); break; | ||
1719 | case IR_TOSTR: asm_tostr(as, ir); break; | ||
1720 | case IR_STRTO: asm_strto(as, ir); break; | ||
1721 | |||
1722 | /* Calls. */ | ||
1723 | case IR_CALLN: case IR_CALLL: case IR_CALLS: asm_call(as, ir); break; | ||
1724 | case IR_CALLXS: asm_callx(as, ir); break; | ||
1725 | case IR_CARG: break; | ||
1726 | |||
1727 | default: | ||
1728 | setintV(&as->J->errinfo, ir->o); | ||
1729 | lj_trace_err_info(as->J, LJ_TRERR_NYIIR); | ||
1730 | break; | ||
1731 | } | ||
1732 | } | ||
1733 | |||
1734 | /* -- Trace setup --------------------------------------------------------- */ | ||
1735 | |||
1736 | /* Ensure there are enough stack slots for call arguments. */ | ||
1737 | static Reg asm_setup_call_slots(ASMState *as, IRIns *ir, const CCallInfo *ci) | ||
1738 | { | ||
1739 | IRRef args[CCI_NARGS_MAX]; | ||
1740 | uint32_t i, nargs = (int)CCI_NARGS(ci); | ||
1741 | int nslots = 0, ngpr = REGARG_NUMGPR; | ||
1742 | asm_collectargs(as, ir, ci, args); | ||
1743 | for (i = 0; i < nargs; i++) | ||
1744 | if (!LJ_SOFTFP && args[i] && irt_isnum(IR(args[i])->t)) { | ||
1745 | ngpr &= ~1; | ||
1746 | if (ngpr > 0) ngpr -= 2; else nslots += 2; | ||
1747 | } else { | ||
1748 | if (ngpr > 0) ngpr--; else nslots++; | ||
1749 | } | ||
1750 | if (nslots > as->evenspill) /* Leave room for args in stack slots. */ | ||
1751 | as->evenspill = nslots; | ||
1752 | return REGSP_HINT(RID_RET); | ||
1753 | } | ||
1754 | |||
1755 | static void asm_setup_target(ASMState *as) | ||
1756 | { | ||
1757 | /* May need extra exit for asm_stack_check on side traces. */ | ||
1758 | asm_exitstub_setup(as, as->T->nsnap + (as->parent ? 1 : 0)); | ||
1759 | } | ||
1760 | |||
1761 | /* -- Trace patching ------------------------------------------------------ */ | ||
1762 | |||
1763 | /* Patch exit jumps of existing machine code to a new target. */ | ||
1764 | void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target) | ||
1765 | { | ||
1766 | MCode *p = T->mcode; | ||
1767 | MCode *pe = (MCode *)((char *)p + T->szmcode); | ||
1768 | MCode *cstart = NULL, *cend = p; | ||
1769 | MCode *mcarea = lj_mcode_patch(J, p, 0); | ||
1770 | MCode *px = exitstub_addr(J, exitno) - 2; | ||
1771 | for (; p < pe; p++) { | ||
1772 | /* Look for bl_cc exitstub, replace with b_cc target. */ | ||
1773 | uint32_t ins = *p; | ||
1774 | if ((ins & 0x0f000000u) == 0x0b000000u && ins < 0xf0000000u && | ||
1775 | ((ins ^ (px-p)) & 0x00ffffffu) == 0) { | ||
1776 | *p = (ins & 0xfe000000u) | (((target-p)-2) & 0x00ffffffu); | ||
1777 | cend = p+1; | ||
1778 | if (!cstart) cstart = p; | ||
1779 | } | ||
1780 | } | ||
1781 | lua_assert(cstart != NULL); | ||
1782 | lj_mcode_sync(cstart, cend); | ||
1783 | lj_mcode_patch(J, mcarea, 1); | ||
1784 | } | ||
1785 | |||