/* Narrowing of power operator or math.pow. */
TRef lj_opt_narrow_pow(jit_State *J, TRef rb, TRef rc, TValue *vc)
{
lua_Number n;
if (tvisstr(vc) && !lj_str_tonum(strV(vc), vc))
lj_trace_err(J, LJ_TRERR_BADTYPE);
n = numV(vc);
/* Limit narrowing for pow to small exponents (or for two constants). */
if ((tref_isk(rc) && tref_isint(rc) && tref_isk(rb)) ||
((J->flags & JIT_F_OPT_NARROW) &&
(numisint(n) && n >= -65536.0 && n <= 65536.0))) {
TRef tmp;
if (!tref_isinteger(rc)) {
if (tref_isstr(rc))
rc = emitir(IRTG(IR_STRTO, IRT_NUM), rc, 0);
rc = emitir(IRTGI(IR_TOINT), rc, IRTOINT_CHECK); /* Guarded TOINT! */
}
if (!tref_isk(rc)) { /* Range guard: -65536 <= i <= 65536 */
tmp = emitir(IRTI(IR_ADD), rc, lj_ir_kint(J, 65536-2147483647-1));
emitir(IRTGI(IR_LE), tmp, lj_ir_kint(J, 2*65536-2147483647-1));
}
return emitir(IRTN(IR_POWI), rb, rc);
}
/* FOLD covers most cases, but some are easier to do here. */
if (tref_isk(rb) && tvispone(ir_knum(IR(tref_ref(rb)))))
return rb; /* 1 ^ x ==> 1 */
rc = lj_ir_tonum(J, rc);
if (tref_isk(rc) && ir_knum(IR(tref_ref(rc)))->n == 0.5)
return emitir(IRTN(IR_FPMATH), rb, IRFPM_SQRT); /* x ^ 0.5 ==> sqrt(x) */
/* Split up b^c into exp2(c*log2(b)). Assembler may rejoin later. */
rb = emitir(IRTN(IR_FPMATH), rb, IRFPM_LOG2);
rc = emitir(IRTN(IR_MUL), rb, rc);
return emitir(IRTN(IR_FPMATH), rc, IRFPM_EXP2);
}
static void LJ_FASTCALL recff_tonumber(jit_State *J, RecordFFData *rd)
{
TRef tr = J->base[0];
TRef base = J->base[1];
if (tr && base) {
base = lj_opt_narrow_toint(J, base);
if (!tref_isk(base) || IR(tref_ref(base))->i != 10)
recff_nyiu(J);
}
if (tref_isnumber_str(tr)) {
if (tref_isstr(tr)) {
TValue tmp;
if (!lj_str_tonum(strV(&rd->argv[0]), &tmp))
recff_nyiu(J); /* Would need an inverted STRTO for this case. */
tr = emitir(IRTG(IR_STRTO, IRT_NUM), tr, 0);
}
#if LJ_HASFFI
} else if (tref_iscdata(tr)) {
lj_crecord_tonumber(J, rd);
return;
#endif
} else {
tr = TREF_NIL;
}
J->base[0] = tr;
UNUSED(rd);
}
/* Determine mode of select() call. */
int32_t lj_ffrecord_select_mode(jit_State *J, TRef tr, TValue *tv)
{
if (tref_isstr(tr) && *strVdata(tv) == '#') { /* select('#', ...) */
if (strV(tv)->len == 1) {
emitir(IRTG(IR_EQ, IRT_STR), tr, lj_ir_kstr(J, strV(tv)));
} else {
TRef trptr = emitir(IRT(IR_STRREF, IRT_P32), tr, lj_ir_kint(J, 0));
TRef trchar = emitir(IRT(IR_XLOAD, IRT_U8), trptr, IRXLOAD_READONLY);
emitir(IRTG(IR_EQ, IRT_INT), trchar, lj_ir_kint(J, '#'));
}
return 0;
} else { /* select(n, ...) */
int32_t start = argv2int(J, tv);
if (start == 0) lj_trace_err(J, LJ_TRERR_BADTYPE); /* A bit misleading. */
return start;
}
}
static TRef crec_tv_ct(jit_State *J, CType *s, CTypeID sid, TRef sp)
{
CTState *cts = ctype_ctsG(J2G(J));
CTInfo sinfo = s->info;
lua_assert(!ctype_isenum(sinfo));
if (ctype_isnum(sinfo)) {
IRType t = crec_ct2irt(s);
TRef tr;
if (t == IRT_CDATA)
goto err_nyi; /* NYI: copyval of >64 bit integers. */
tr = emitir(IRT(IR_XLOAD, t), sp, 0);
if (t == IRT_FLOAT || t == IRT_U32) { /* Keep uint32_t/float as numbers. */
tr = emitconv(tr, IRT_NUM, t, 0);
} else if (t == IRT_I64 || t == IRT_U64) { /* Box 64 bit integer. */
TRef dp = emitir(IRTG(IR_CNEW, IRT_CDATA), lj_ir_kint(J, sid), TREF_NIL);
TRef ptr = emitir(IRT(IR_ADD, IRT_PTR), dp,
lj_ir_kintp(J, sizeof(GCcdata)));
emitir(IRT(IR_XSTORE, t), ptr, tr);
return dp;
} else if ((sinfo & CTF_BOOL)) {
/* Assume not equal to zero. Fixup and emit pending guard later. */
lj_ir_set(J, IRTGI(IR_NE), tr, lj_ir_kint(J, 0));
J->postproc = LJ_POST_FIXGUARD;
tr = TREF_TRUE;
}
return tr;
} else if (ctype_isptr(sinfo)) {
IRType t = (LJ_64 && s->size == 8) ? IRT_P64 : IRT_P32;
sp = emitir(IRT(IR_XLOAD, t), sp, 0);
} else if (ctype_isrefarray(sinfo) || ctype_isstruct(sinfo)) {
cts->L = J->L;
sid = lj_ctype_intern(cts, CTINFO_REF(sid), CTSIZE_PTR); /* Create ref. */
} else if (ctype_iscomplex(sinfo)) { /* Unbox/box complex. */
IRType t = s->size == 2*sizeof(double) ? IRT_NUM : IRT_FLOAT;
ptrdiff_t esz = (ptrdiff_t)(s->size >> 1);
TRef ptr, tr1, tr2, dp;
dp = emitir(IRTG(IR_CNEW, IRT_CDATA), lj_ir_kint(J, sid), TREF_NIL);
tr1 = emitir(IRT(IR_XLOAD, t), sp, 0);
ptr = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, esz));
tr2 = emitir(IRT(IR_XLOAD, t), ptr, 0);
ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, sizeof(GCcdata)));
emitir(IRT(IR_XSTORE, t), ptr, tr1);
ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, sizeof(GCcdata)+esz));
emitir(IRT(IR_XSTORE, t), ptr, tr2);
return dp;
} else {
/* Specialize to the CTypeID held by a cdata constructor. */
static CTypeID crec_constructor(jit_State *J, GCcdata *cd, TRef tr)
{
CTypeID id;
lua_assert(tref_iscdata(tr) && cd->typeid == CTID_CTYPEID);
id = *(CTypeID *)cdataptr(cd);
tr = emitir(IRT(IR_ADD, IRT_PTR), tr, lj_ir_kintp(J, sizeof(GCcdata)));
tr = emitir(IRT(IR_XLOAD, IRT_INT), tr, 0);
emitir(IRTG(IR_EQ, IRT_INT), tr, lj_ir_kint(J, (int32_t)id));
return id;
}
/* Emit the conversions collected during backpropagation. */
static IRRef narrow_conv_emit(jit_State *J, NarrowConv *nc)
{
/* The fins fields must be saved now -- emitir() overwrites them. */
IROpT guardot = irt_isguard(fins->t) ? IRTG(IR_ADDOV-IR_ADD, 0) : 0;
IROpT convot = fins->ot;
IRRef1 convop2 = fins->op2;
NarrowIns *next = nc->stack; /* List of instructions from backpropagation. */
NarrowIns *last = nc->sp;
NarrowIns *sp = nc->stack; /* Recycle the stack to store operands. */
while (next < last) { /* Simple stack machine to process the ins. list. */
NarrowIns ref = *next++;
IROpT op = narrow_op(ref);
if (op == NARROW_REF) {
*sp++ = ref;
} else if (op == NARROW_CONV) {
*sp++ = emitir_raw(convot, ref, convop2); /* Raw emit avoids a loop. */
} else if (op == NARROW_SEXT) {
lua_assert(sp >= nc->stack+1);
sp[-1] = emitir(IRT(IR_CONV, IRT_I64), sp[-1],
(IRT_I64<<5)|IRT_INT|IRCONV_SEXT);
} else if (op == NARROW_INT) {
lua_assert(next < last);
*sp++ = nc->t == IRT_I64 ?
lj_ir_kint64(J, (int64_t)(int32_t)*next++) :
lj_ir_kint(J, *next++);
} else { /* Regular IROpT. Pops two operands and pushes one result. */
IRRef mode = nc->mode;
lua_assert(sp >= nc->stack+2);
sp--;
/* Omit some overflow checks for array indexing. See comments above. */
if ((mode & IRCONV_CONVMASK) == IRCONV_INDEX) {
if (next == last && irref_isk(narrow_ref(sp[0])) &&
(uint32_t)IR(narrow_ref(sp[0]))->i + 0x40000000u < 0x80000000u)
guardot = 0;
else /* Otherwise cache a stronger check. */
mode += IRCONV_CHECK-IRCONV_INDEX;
}
sp[-1] = emitir(op+guardot, sp[-1], sp[0]);
/* Add to cache. */
if (narrow_ref(ref))
narrow_bpc_set(J, narrow_ref(ref), narrow_ref(sp[-1]), mode);
}
}
lua_assert(sp == nc->stack+1);
return nc->stack[0];
}
/* Get FILE* for I/O function. Any I/O error aborts recording, so there's
** no need to encode the alternate cases for any of the guards.
*/
static TRef recff_io_fp(jit_State *J, uint32_t id)
{
TRef tr, ud, fp;
if (id) { /* io.func() */
tr = lj_ir_kptr(J, &J2G(J)->gcroot[id]);
ud = emitir(IRT(IR_XLOAD, IRT_UDATA), tr, 0);
} else { /* fp:method() */
ud = J->base[0];
if (!tref_isudata(ud))
lj_trace_err(J, LJ_TRERR_BADTYPE);
tr = emitir(IRT(IR_FLOAD, IRT_U8), ud, IRFL_UDATA_UDTYPE);
emitir(IRTGI(IR_EQ), tr, lj_ir_kint(J, UDTYPE_IO_FILE));
}
fp = emitir(IRT(IR_FLOAD, IRT_PTR), ud, IRFL_UDATA_FILE);
emitir(IRTG(IR_NE, IRT_PTR), fp, lj_ir_knull(J, IRT_PTR));
return fp;
}
/* Emit a (checked) number to integer conversion. */
static IRRef split_num2int(jit_State *J, IRRef lo, IRRef hi, int check)
{
IRRef tmp, res;
#if LJ_LE
tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), lo, hi);
#else
tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), hi, lo);
#endif
res = split_emit(J, IRTI(IR_CALLN), tmp, IRCALL_softfp_d2i);
if (check) {
tmp = split_emit(J, IRTI(IR_CALLN), res, IRCALL_softfp_i2d);
split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), tmp, tmp);
split_emit(J, IRTGI(IR_EQ), tmp, lo);
split_emit(J, IRTG(IR_HIOP, IRT_SOFTFP), tmp+1, hi);
}
return res;
}
static CTypeID argv2ctype(jit_State *J, TRef tr, cTValue *o)
{
if (tref_isstr(tr)) {
GCstr *s = strV(o);
CPState cp;
CTypeID oldtop;
/* Specialize to the string containing the C type declaration. */
emitir(IRTG(IR_EQ, IRT_STR), tr, lj_ir_kstr(J, s));
cp.L = J->L;
cp.cts = ctype_ctsG(J2G(J));
oldtop = cp.cts->top;
cp.srcname = strdata(s);
cp.p = strdata(s);
cp.mode = CPARSE_MODE_ABSTRACT|CPARSE_MODE_NOIMPLICIT;
if (lj_cparse(&cp) || cp.cts->top > oldtop) /* Avoid new struct defs. */
lj_trace_err(J, LJ_TRERR_BADTYPE);
return cp.val.id;
} else {
GCcdata *cd = argv2cdata(J, tr, o);
return cd->typeid == CTID_CTYPEID ? *(CTypeID *)cdataptr(cd) : cd->typeid;
}
}
/* Unroll loop. */
static void loop_unroll(jit_State *J)
{
IRRef1 phi[LJ_MAX_PHI];
uint32_t nphi = 0;
IRRef1 *subst;
SnapNo onsnap;
SnapShot *osnap, *loopsnap;
SnapEntry *loopmap, *psentinel;
IRRef ins, invar;
/* Use temp buffer for substitution table.
** Only non-constant refs in [REF_BIAS,invar) are valid indexes.
** Caveat: don't call into the VM or run the GC or the buffer may be gone.
*/
invar = J->cur.nins;
subst = (IRRef1 *)lj_str_needbuf(J->L, &G(J->L)->tmpbuf,
(invar-REF_BIAS)*sizeof(IRRef1)) - REF_BIAS;
subst[REF_BASE] = REF_BASE;
/* LOOP separates the pre-roll from the loop body. */
emitir_raw(IRTG(IR_LOOP, IRT_NIL), 0, 0);
/* Grow snapshot buffer and map for copy-substituted snapshots.
** Need up to twice the number of snapshots minus #0 and loop snapshot.
** Need up to twice the number of entries plus fallback substitutions
** from the loop snapshot entries for each new snapshot.
** Caveat: both calls may reallocate J->cur.snap and J->cur.snapmap!
*/
onsnap = J->cur.nsnap;
lj_snap_grow_buf(J, 2*onsnap-2);
lj_snap_grow_map(J, J->cur.nsnapmap*2+(onsnap-2)*J->cur.snap[onsnap-1].nent);
/* The loop snapshot is used for fallback substitutions. */
loopsnap = &J->cur.snap[onsnap-1];
loopmap = &J->cur.snapmap[loopsnap->mapofs];
/* The PC of snapshot #0 and the loop snapshot must match. */
psentinel = &loopmap[loopsnap->nent];
lua_assert(*psentinel == J->cur.snapmap[J->cur.snap[0].nent]);
*psentinel = SNAP(255, 0, 0); /* Replace PC with temporary sentinel. */
/* Start substitution with snapshot #1 (#0 is empty for root traces). */
osnap = &J->cur.snap[1];
/* Copy and substitute all recorded instructions and snapshots. */
for (ins = REF_FIRST; ins < invar; ins++) {
IRIns *ir;
IRRef op1, op2;
if (ins >= osnap->ref) /* Instruction belongs to next snapshot? */
loop_subst_snap(J, osnap++, loopmap, subst); /* Copy-substitute it. */
/* Substitute instruction operands. */
ir = IR(ins);
op1 = ir->op1;
if (!irref_isk(op1)) op1 = subst[op1];
op2 = ir->op2;
if (!irref_isk(op2)) op2 = subst[op2];
if (irm_kind(lj_ir_mode[ir->o]) == IRM_N &&
op1 == ir->op1 && op2 == ir->op2) { /* Regular invariant ins? */
subst[ins] = (IRRef1)ins; /* Shortcut. */
} else {
/* Re-emit substituted instruction to the FOLD/CSE/etc. pipeline. */
IRType1 t = ir->t; /* Get this first, since emitir may invalidate ir. */
IRRef ref = tref_ref(emitir(ir->ot & ~IRT_ISPHI, op1, op2));
subst[ins] = (IRRef1)ref;
if (ref != ins) {
IRIns *irr = IR(ref);
if (ref < invar) { /* Loop-carried dependency? */
/* Potential PHI? */
if (!irref_isk(ref) && !irt_isphi(irr->t) && !irt_ispri(irr->t)) {
irt_setphi(irr->t);
if (nphi >= LJ_MAX_PHI)
lj_trace_err(J, LJ_TRERR_PHIOV);
phi[nphi++] = (IRRef1)ref;
}
/* Check all loop-carried dependencies for type instability. */
if (!irt_sametype(t, irr->t)) {
if (irt_isinteger(t) && irt_isinteger(irr->t))
continue;
else if (irt_isnum(t) && irt_isinteger(irr->t)) /* Fix int->num. */
ref = tref_ref(emitir(IRTN(IR_CONV), ref, IRCONV_NUM_INT));
else if (irt_isnum(irr->t) && irt_isinteger(t)) /* Fix num->int. */
ref = tref_ref(emitir(IRTGI(IR_CONV), ref,
IRCONV_INT_NUM|IRCONV_CHECK));
else
lj_trace_err(J, LJ_TRERR_TYPEINS);
subst[ins] = (IRRef1)ref;
irr = IR(ref);
goto phiconv;
}
} else if (ref != REF_DROP && irr->o == IR_CONV &&
ref > invar && irr->op1 < invar) {
/* May need an extra PHI for a CONV. */
ref = irr->op1;
irr = IR(ref);
phiconv:
if (ref < invar && !irref_isk(ref) && !irt_isphi(irr->t)) {
irt_setphi(irr->t);
if (nphi >= LJ_MAX_PHI)
lj_trace_err(J, LJ_TRERR_PHIOV);
phi[nphi++] = (IRRef1)ref;
}
}
}
}
}
if (!irt_isguard(J->guardemit)) /* Drop redundant snapshot. */
J->cur.nsnapmap = (uint16_t)J->cur.snap[--J->cur.nsnap].mapofs;
lua_assert(J->cur.nsnapmap <= J->sizesnapmap);
*psentinel = J->cur.snapmap[J->cur.snap[0].nent]; /* Restore PC. */
loop_emit_phi(J, subst, phi, nphi, onsnap);
}
static void crec_ct_ct(jit_State *J, CType *d, CType *s, TRef dp, TRef sp,
void *svisnz)
{
CTSize dsize = d->size, ssize = s->size;
CTInfo dinfo = d->info, sinfo = s->info;
IRType dt = crec_ct2irt(d);
IRType st = crec_ct2irt(s);
if (ctype_type(dinfo) > CT_MAYCONVERT || ctype_type(sinfo) > CT_MAYCONVERT)
goto err_conv;
/*
** Note: Unlike lj_cconv_ct_ct(), sp holds the _value_ of pointers and
** numbers up to 8 bytes. Otherwise sp holds a pointer.
*/
switch (cconv_idx2(dinfo, sinfo)) {
/* Destination is a bool. */
case CCX(B, B):
goto xstore; /* Source operand is already normalized. */
case CCX(B, I):
case CCX(B, F):
if (st != IRT_CDATA) {
/* Specialize to the result of a comparison against 0. */
TRef zero = (st == IRT_NUM || st == IRT_FLOAT) ? lj_ir_knum(J, 0) :
(st == IRT_I64 || st == IRT_U64) ? lj_ir_kint64(J, 0) :
lj_ir_kint(J, 0);
int isnz = crec_isnonzero(s, svisnz);
emitir(IRTG(isnz ? IR_NE : IR_EQ, st), sp, zero);
sp = lj_ir_kint(J, isnz);
goto xstore;
}
goto err_nyi;
/* Destination is an integer. */
case CCX(I, B):
case CCX(I, I):
conv_I_I:
if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
#if LJ_64
/* Sign-extend 32 to 64 bit integer. */
if (dsize == 8 && ssize < 8 && !(sinfo & CTF_UNSIGNED))
sp = emitconv(sp, dt, IRT_INT, IRCONV_SEXT);
/* All other conversions are no-ops on x64. */
#else
if (dsize == 8 && ssize < 8) /* Extend to 64 bit integer. */
sp = emitconv(sp, dt, ssize < 4 ? IRT_INT : st,
(sinfo & CTF_UNSIGNED) ? 0 : IRCONV_SEXT);
else if (dsize < 8 && ssize == 8) /* Truncate from 64 bit integer. */
sp = emitconv(sp, dsize < 4 ? IRT_INT : dt, st, 0);
#endif
xstore:
emitir(IRT(IR_XSTORE, dt), dp, sp);
break;
case CCX(I, C):
sp = emitir(IRT(IR_XLOAD, st), sp, 0); /* Load re. */
/* fallthrough */
case CCX(I, F):
if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
sp = emitconv(sp, dsize < 4 ? IRT_INT : dt, st, IRCONV_TRUNC|IRCONV_ANY);
goto xstore;
case CCX(I, P):
case CCX(I, A):
sinfo = CTINFO(CT_NUM, CTF_UNSIGNED);
ssize = CTSIZE_PTR;
st = IRT_UINTP;
goto conv_I_I;
/* Destination is a floating-point number. */
case CCX(F, B):
case CCX(F, I):
conv_F_I:
if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
sp = emitconv(sp, dt, ssize < 4 ? IRT_INT : st, 0);
goto xstore;
case CCX(F, C):
sp = emitir(IRT(IR_XLOAD, st), sp, 0); /* Load re. */
/* fallthrough */
case CCX(F, F):
conv_F_F:
if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
if (dt != st) sp = emitconv(sp, dt, st, 0);
goto xstore;
/* Destination is a complex number. */
case CCX(C, I):
case CCX(C, F):
{ /* Clear im. */
TRef ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, (dsize >> 1)));
emitir(IRT(IR_XSTORE, dt), ptr, lj_ir_knum(J, 0));
}
/* Convert to re. */
if ((sinfo & CTF_FP)) goto conv_F_F; else goto conv_F_I;
case CCX(C, C):
if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
{
TRef re, im, ptr;
re = emitir(IRT(IR_XLOAD, st), sp, 0);
ptr = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, (ssize >> 1)));
im = emitir(IRT(IR_XLOAD, st), ptr, 0);
if (dt != st) {
re = emitconv(re, dt, st, 0);
im = emitconv(im, dt, st, 0);
}
emitir(IRT(IR_XSTORE, dt), dp, re);
ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, (dsize >> 1)));
emitir(IRT(IR_XSTORE, dt), ptr, im);
}
break;
/* Destination is a vector. */
case CCX(V, I):
case CCX(V, F):
case CCX(V, C):
case CCX(V, V):
goto err_nyi;
/* Destination is a pointer. */
case CCX(P, P):
case CCX(P, A):
case CCX(P, S):
/* There are only 32 bit pointers/addresses on 32 bit machines.
** Also ok on x64, since all 32 bit ops clear the upper part of the reg.
*/
goto xstore;
case CCX(P, I):
if (st == IRT_CDATA) goto err_nyi;
if (!LJ_64 && ssize == 8) /* Truncate from 64 bit integer. */
sp = emitconv(sp, IRT_U32, st, 0);
goto xstore;
case CCX(P, F):
if (st == IRT_CDATA) goto err_nyi;
/* The signed conversion is cheaper. x64 really has 47 bit pointers. */
sp = emitconv(sp, (LJ_64 && dsize == 8) ? IRT_I64 : IRT_U32,
st, IRCONV_TRUNC|IRCONV_ANY);
goto xstore;
/* Destination is an array. */
case CCX(A, A):
goto err_nyi;
/* Destination is a struct/union. */
case CCX(S, S):
goto err_nyi;
default:
err_conv:
err_nyi:
lj_trace_err(J, LJ_TRERR_NYICONV);
break;
}
}
/* Replay snapshot state to setup side trace. */
void lj_snap_replay(jit_State *J, GCtrace *T)
{
SnapShot *snap = &T->snap[J->exitno];
SnapEntry *map = &T->snapmap[snap->mapofs];
MSize n, nent = snap->nent;
BloomFilter seen = 0;
int pass23 = 0;
J->framedepth = 0;
/* Emit IR for slots inherited from parent snapshot. */
for (n = 0; n < nent; n++) {
SnapEntry sn = map[n];
BCReg s = snap_slot(sn);
IRRef ref = snap_ref(sn);
IRIns *ir = &T->ir[ref];
TRef tr;
/* The bloom filter avoids O(nent^2) overhead for de-duping slots. */
if (bloomtest(seen, ref) && (tr = snap_dedup(J, map, n, ref)) != 0)
goto setslot;
bloomset(seen, ref);
if (irref_isk(ref)) {
tr = snap_replay_const(J, ir);
} else if (!regsp_used(ir->prev)) {
pass23 = 1;
lua_assert(s != 0);
tr = s;
} else {
IRType t = irt_type(ir->t);
uint32_t mode = IRSLOAD_INHERIT|IRSLOAD_PARENT;
if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) t = IRT_NUM;
if (ir->o == IR_SLOAD) mode |= (ir->op2 & IRSLOAD_READONLY);
tr = emitir_raw(IRT(IR_SLOAD, t), s, mode);
}
setslot:
J->slot[s] = tr | (sn&(SNAP_CONT|SNAP_FRAME)); /* Same as TREF_* flags. */
J->framedepth += ((sn & (SNAP_CONT|SNAP_FRAME)) && s);
if ((sn & SNAP_FRAME))
J->baseslot = s+1;
}
if (pass23) {
IRIns *irlast = &T->ir[snap->ref];
pass23 = 0;
/* Emit dependent PVALs. */
for (n = 0; n < nent; n++) {
SnapEntry sn = map[n];
IRRef refp = snap_ref(sn);
IRIns *ir = &T->ir[refp];
if (regsp_reg(ir->r) == RID_SUNK) {
if (J->slot[snap_slot(sn)] != snap_slot(sn)) continue;
pass23 = 1;
lua_assert(ir->o == IR_TNEW || ir->o == IR_TDUP ||
ir->o == IR_CNEW || ir->o == IR_CNEWI);
if (ir->op1 >= T->nk) snap_pref(J, T, map, nent, seen, ir->op1);
if (ir->op2 >= T->nk) snap_pref(J, T, map, nent, seen, ir->op2);
if (LJ_HASFFI && ir->o == IR_CNEWI) {
if (LJ_32 && refp+1 < T->nins && (ir+1)->o == IR_HIOP)
snap_pref(J, T, map, nent, seen, (ir+1)->op2);
} else {
IRIns *irs;
for (irs = ir+1; irs < irlast; irs++)
if (irs->r == RID_SINK && snap_sunk_store(T, ir, irs)) {
if (snap_pref(J, T, map, nent, seen, irs->op2) == 0)
snap_pref(J, T, map, nent, seen, T->ir[irs->op2].op1);
else if ((LJ_SOFTFP || (LJ_32 && LJ_HASFFI)) &&
irs+1 < irlast && (irs+1)->o == IR_HIOP)
snap_pref(J, T, map, nent, seen, (irs+1)->op2);
}
}
} else if (!irref_isk(refp) && !regsp_used(ir->prev)) {
lua_assert(ir->o == IR_CONV && ir->op2 == IRCONV_NUM_INT);
J->slot[snap_slot(sn)] = snap_pref(J, T, map, nent, seen, ir->op1);
}
}
/* Replay sunk instructions. */
for (n = 0; pass23 && n < nent; n++) {
SnapEntry sn = map[n];
IRRef refp = snap_ref(sn);
IRIns *ir = &T->ir[refp];
if (regsp_reg(ir->r) == RID_SUNK) {
TRef op1, op2;
if (J->slot[snap_slot(sn)] != snap_slot(sn)) { /* De-dup allocs. */
J->slot[snap_slot(sn)] = J->slot[J->slot[snap_slot(sn)]];
continue;
}
op1 = ir->op1;
if (op1 >= T->nk) op1 = snap_pref(J, T, map, nent, seen, op1);
op2 = ir->op2;
if (op2 >= T->nk) op2 = snap_pref(J, T, map, nent, seen, op2);
if (LJ_HASFFI && ir->o == IR_CNEWI) {
if (LJ_32 && refp+1 < T->nins && (ir+1)->o == IR_HIOP) {
lj_needsplit(J); /* Emit joining HIOP. */
op2 = emitir_raw(IRT(IR_HIOP, IRT_I64), op2,
snap_pref(J, T, map, nent, seen, (ir+1)->op2));
}
J->slot[snap_slot(sn)] = emitir(ir->ot & ~(IRT_MARK|IRT_ISPHI), op1, op2);
} else {
IRIns *irs;
TRef tr = emitir(ir->ot, op1, op2);
J->slot[snap_slot(sn)] = tr;
for (irs = ir+1; irs < irlast; irs++)
if (irs->r == RID_SINK && snap_sunk_store(T, ir, irs)) {
IRIns *irr = &T->ir[irs->op1];
TRef val, key = irr->op2, tmp = tr;
if (irr->o != IR_FREF) {
IRIns *irk = &T->ir[key];
if (irr->o == IR_HREFK)
key = lj_ir_kslot(J, snap_replay_const(J, &T->ir[irk->op1]),
irk->op2);
else
key = snap_replay_const(J, irk);
if (irr->o == IR_HREFK || irr->o == IR_AREF) {
IRIns *irf = &T->ir[irr->op1];
tmp = emitir(irf->ot, tmp, irf->op2);
}
}
tmp = emitir(irr->ot, tmp, key);
val = snap_pref(J, T, map, nent, seen, irs->op2);
if (val == 0) {
IRIns *irc = &T->ir[irs->op2];
lua_assert(irc->o == IR_CONV && irc->op2 == IRCONV_NUM_INT);
val = snap_pref(J, T, map, nent, seen, irc->op1);
val = emitir(IRTN(IR_CONV), val, IRCONV_NUM_INT);
} else if ((LJ_SOFTFP || (LJ_32 && LJ_HASFFI)) &&
irs+1 < irlast && (irs+1)->o == IR_HIOP) {
IRType t = IRT_I64;
if (LJ_SOFTFP && irt_type((irs+1)->t) == IRT_SOFTFP)
t = IRT_NUM;
lj_needsplit(J);
if (irref_isk(irs->op2) && irref_isk((irs+1)->op2)) {
uint64_t k = (uint32_t)T->ir[irs->op2].i +
((uint64_t)T->ir[(irs+1)->op2].i << 32);
val = lj_ir_k64(J, t == IRT_I64 ? IR_KINT64 : IR_KNUM,
lj_ir_k64_find(J, k));
} else {
val = emitir_raw(IRT(IR_HIOP, t), val,
snap_pref(J, T, map, nent, seen, (irs+1)->op2));
}
tmp = emitir(IRT(irs->o, t), tmp, val);
continue;
}
tmp = emitir(irs->ot, tmp, val);
} else if (LJ_HASFFI && irs->o == IR_XBAR && ir->o == IR_CNEW) {
emitir(IRT(IR_XBAR, IRT_NIL), 0, 0);
}
}
}
}
}
J->base = J->slot + J->baseslot;
J->maxslot = snap->nslots - J->baseslot;
lj_snap_add(J);
if (pass23) /* Need explicit GC step _after_ initial snapshot. */
emitir_raw(IRTG(IR_GCSTEP, IRT_NIL), 0, 0);
}
/* Transform the old IR to the new IR. */
static void split_ir(jit_State *J)
{
IRRef nins = J->cur.nins, nk = J->cur.nk;
MSize irlen = nins - nk;
MSize need = (irlen+1)*(sizeof(IRIns) + sizeof(IRRef1));
IRIns *oir = (IRIns *)lj_str_needbuf(J->L, &G(J->L)->tmpbuf, need);
IRRef1 *hisubst;
IRRef ref, snref;
SnapShot *snap;
/* Copy old IR to buffer. */
memcpy(oir, IR(nk), irlen*sizeof(IRIns));
/* Bias hiword substitution table and old IR. Loword kept in field prev. */
hisubst = (IRRef1 *)&oir[irlen] - nk;
oir -= nk;
/* Remove all IR instructions, but retain IR constants. */
J->cur.nins = REF_FIRST;
J->loopref = 0;
/* Process constants and fixed references. */
for (ref = nk; ref <= REF_BASE; ref++) {
IRIns *ir = &oir[ref];
if ((LJ_SOFTFP && ir->o == IR_KNUM) || ir->o == IR_KINT64) {
/* Split up 64 bit constant. */
TValue tv = *ir_k64(ir);
ir->prev = lj_ir_kint(J, (int32_t)tv.u32.lo);
hisubst[ref] = lj_ir_kint(J, (int32_t)tv.u32.hi);
} else {
ir->prev = ref; /* Identity substitution for loword. */
hisubst[ref] = 0;
}
}
/* Process old IR instructions. */
snap = J->cur.snap;
snref = snap->ref;
for (ref = REF_FIRST; ref < nins; ref++) {
IRIns *ir = &oir[ref];
IRRef nref = lj_ir_nextins(J);
IRIns *nir = IR(nref);
IRRef hi = 0;
if (ref >= snref) {
snap->ref = nref;
split_subst_snap(J, snap++, oir);
snref = snap < &J->cur.snap[J->cur.nsnap] ? snap->ref : ~(IRRef)0;
}
/* Copy-substitute old instruction to new instruction. */
nir->op1 = ir->op1 < nk ? ir->op1 : oir[ir->op1].prev;
nir->op2 = ir->op2 < nk ? ir->op2 : oir[ir->op2].prev;
ir->prev = nref; /* Loword substitution. */
nir->o = ir->o;
nir->t.irt = ir->t.irt & ~(IRT_MARK|IRT_ISPHI);
hisubst[ref] = 0;
/* Split 64 bit instructions. */
#if LJ_SOFTFP
if (irt_isnum(ir->t)) {
nir->t.irt = IRT_INT | (nir->t.irt & IRT_GUARD); /* Turn into INT op. */
/* Note: hi ref = lo ref + 1! Required for SNAP_SOFTFPNUM logic. */
switch (ir->o) {
case IR_ADD:
hi = split_call_ll(J, hisubst, oir, ir, IRCALL_softfp_add);
break;
case IR_SUB:
hi = split_call_ll(J, hisubst, oir, ir, IRCALL_softfp_sub);
break;
case IR_MUL:
hi = split_call_ll(J, hisubst, oir, ir, IRCALL_softfp_mul);
break;
case IR_DIV:
hi = split_call_ll(J, hisubst, oir, ir, IRCALL_softfp_div);
break;
case IR_POW:
hi = split_call_li(J, hisubst, oir, ir, IRCALL_lj_vm_powi);
break;
case IR_FPMATH:
/* Try to rejoin pow from EXP2, MUL and LOG2. */
if (nir->op2 == IRFPM_EXP2 && nir->op1 > J->loopref) {
IRIns *irp = IR(nir->op1);
if (irp->o == IR_CALLN && irp->op2 == IRCALL_softfp_mul) {
IRIns *irm4 = IR(irp->op1);
IRIns *irm3 = IR(irm4->op1);
IRIns *irm12 = IR(irm3->op1);
IRIns *irl1 = IR(irm12->op1);
if (irm12->op1 > J->loopref && irl1->o == IR_CALLN &&
irl1->op2 == IRCALL_lj_vm_log2) {
IRRef tmp = irl1->op1; /* Recycle first two args from LOG2. */
IRRef arg3 = irm3->op2, arg4 = irm4->op2;
J->cur.nins--;
tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), tmp, arg3);
tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), tmp, arg4);
ir->prev = tmp = split_emit(J, IRTI(IR_CALLN), tmp, IRCALL_pow);
hi = split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), tmp, tmp);
break;
}
}
}
hi = split_call_l(J, hisubst, oir, ir, IRCALL_lj_vm_floor + ir->op2);
break;
case IR_ATAN2:
hi = split_call_ll(J, hisubst, oir, ir, IRCALL_atan2);
break;
case IR_LDEXP:
hi = split_call_li(J, hisubst, oir, ir, IRCALL_ldexp);
break;
case IR_NEG: case IR_ABS:
nir->o = IR_CONV; /* Pass through loword. */
nir->op2 = (IRT_INT << 5) | IRT_INT;
hi = split_emit(J, IRT(ir->o == IR_NEG ? IR_BXOR : IR_BAND, IRT_SOFTFP),
hisubst[ir->op1], hisubst[ir->op2]);
break;
case IR_SLOAD:
if ((nir->op2 & IRSLOAD_CONVERT)) { /* Convert from int to number. */
nir->op2 &= ~IRSLOAD_CONVERT;
ir->prev = nref = split_emit(J, IRTI(IR_CALLN), nref,
IRCALL_softfp_i2d);
hi = split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), nref, nref);
break;
}
/* fallthrough */
case IR_ALOAD: case IR_HLOAD: case IR_ULOAD: case IR_VLOAD:
case IR_STRTO:
hi = split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), nref, nref);
break;
case IR_XLOAD: {
IRIns inslo = *nir; /* Save/undo the emit of the lo XLOAD. */
J->cur.nins--;
hi = split_ptr(J, oir, ir->op1); /* Insert the hiref ADD. */
nref = lj_ir_nextins(J);
nir = IR(nref);
*nir = inslo; /* Re-emit lo XLOAD immediately before hi XLOAD. */
hi = split_emit(J, IRT(IR_XLOAD, IRT_SOFTFP), hi, ir->op2);
#if LJ_LE
ir->prev = nref;
#else
ir->prev = hi; hi = nref;
#endif
break;
}
case IR_ASTORE: case IR_HSTORE: case IR_USTORE: case IR_XSTORE:
split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), nir->op1, hisubst[ir->op2]);
break;
case IR_CONV: { /* Conversion to number. Others handled below. */
IRType st = (IRType)(ir->op2 & IRCONV_SRCMASK);
UNUSED(st);
#if LJ_32 && LJ_HASFFI
if (st == IRT_I64 || st == IRT_U64) {
hi = split_call_l(J, hisubst, oir, ir,
st == IRT_I64 ? IRCALL_fp64_l2d : IRCALL_fp64_ul2d);
break;
}
#endif
lua_assert(st == IRT_INT ||
(LJ_32 && LJ_HASFFI && (st == IRT_U32 || st == IRT_FLOAT)));
nir->o = IR_CALLN;
#if LJ_32 && LJ_HASFFI
nir->op2 = st == IRT_INT ? IRCALL_softfp_i2d :
st == IRT_FLOAT ? IRCALL_softfp_f2d :
IRCALL_softfp_ui2d;
#else
nir->op2 = IRCALL_softfp_i2d;
#endif
hi = split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), nref, nref);
break;
}
case IR_CALLN:
case IR_CALLL:
case IR_CALLS:
case IR_CALLXS:
goto split_call;
case IR_PHI:
if (nir->op1 == nir->op2)
J->cur.nins--; /* Drop useless PHIs. */
if (hisubst[ir->op1] != hisubst[ir->op2])
split_emit(J, IRT(IR_PHI, IRT_SOFTFP),
hisubst[ir->op1], hisubst[ir->op2]);
break;
case IR_HIOP:
J->cur.nins--; /* Drop joining HIOP. */
ir->prev = nir->op1;
hi = nir->op2;
break;
default:
lua_assert(ir->o <= IR_NE || ir->o == IR_MIN || ir->o == IR_MAX);
hi = split_emit(J, IRTG(IR_HIOP, IRT_SOFTFP),
hisubst[ir->op1], hisubst[ir->op2]);
break;
}
} else
#endif
#if LJ_32 && LJ_HASFFI
if (irt_isint64(ir->t)) {
IRRef hiref = hisubst[ir->op1];
nir->t.irt = IRT_INT | (nir->t.irt & IRT_GUARD); /* Turn into INT op. */
switch (ir->o) {
case IR_ADD:
case IR_SUB:
/* Use plain op for hiword if loword cannot produce a carry/borrow. */
if (irref_isk(nir->op2) && IR(nir->op2)->i == 0) {
ir->prev = nir->op1; /* Pass through loword. */
nir->op1 = hiref; nir->op2 = hisubst[ir->op2];
hi = nref;
break;
}
/* fallthrough */
case IR_NEG:
hi = split_emit(J, IRTI(IR_HIOP), hiref, hisubst[ir->op2]);
break;
case IR_MUL:
hi = split_call_ll(J, hisubst, oir, ir, IRCALL_lj_carith_mul64);
break;
case IR_DIV:
hi = split_call_ll(J, hisubst, oir, ir,
irt_isi64(ir->t) ? IRCALL_lj_carith_divi64 :
IRCALL_lj_carith_divu64);
break;
case IR_MOD:
hi = split_call_ll(J, hisubst, oir, ir,
irt_isi64(ir->t) ? IRCALL_lj_carith_modi64 :
IRCALL_lj_carith_modu64);
break;
case IR_POW:
hi = split_call_ll(J, hisubst, oir, ir,
irt_isi64(ir->t) ? IRCALL_lj_carith_powi64 :
IRCALL_lj_carith_powu64);
break;
case IR_FLOAD:
lua_assert(ir->op2 == IRFL_CDATA_INT64);
hi = split_emit(J, IRTI(IR_FLOAD), nir->op1, IRFL_CDATA_INT64_4);
#if LJ_BE
ir->prev = hi; hi = nref;
#endif
break;
case IR_XLOAD:
hi = split_emit(J, IRTI(IR_XLOAD), split_ptr(J, oir, ir->op1), ir->op2);
#if LJ_BE
ir->prev = hi; hi = nref;
#endif
break;
case IR_XSTORE:
split_emit(J, IRTI(IR_HIOP), nir->op1, hisubst[ir->op2]);
break;
case IR_CONV: { /* Conversion to 64 bit integer. Others handled below. */
IRType st = (IRType)(ir->op2 & IRCONV_SRCMASK);
#if LJ_SOFTFP
if (st == IRT_NUM) { /* NUM to 64 bit int conv. */
hi = split_call_l(J, hisubst, oir, ir,
irt_isi64(ir->t) ? IRCALL_fp64_d2l : IRCALL_fp64_d2ul);
} else if (st == IRT_FLOAT) { /* FLOAT to 64 bit int conv. */
nir->o = IR_CALLN;
nir->op2 = irt_isi64(ir->t) ? IRCALL_fp64_f2l : IRCALL_fp64_f2ul;
hi = split_emit(J, IRTI(IR_HIOP), nref, nref);
}
#else
if (st == IRT_NUM || st == IRT_FLOAT) { /* FP to 64 bit int conv. */
hi = split_emit(J, IRTI(IR_HIOP), nir->op1, nref);
}
#endif
else if (st == IRT_I64 || st == IRT_U64) { /* 64/64 bit cast. */
/* Drop cast, since assembler doesn't care. But fwd both parts. */
hi = hiref;
goto fwdlo;
} else if ((ir->op2 & IRCONV_SEXT)) { /* Sign-extend to 64 bit. */
IRRef k31 = lj_ir_kint(J, 31);
nir = IR(nref); /* May have been reallocated. */
ir->prev = nir->op1; /* Pass through loword. */
nir->o = IR_BSAR; /* hi = bsar(lo, 31). */
nir->op2 = k31;
hi = nref;
} else { /* Zero-extend to 64 bit. */
hi = lj_ir_kint(J, 0);
goto fwdlo;
}
break;
}
case IR_CALLXS:
goto split_call;
case IR_PHI: {
IRRef hiref2;
if ((irref_isk(nir->op1) && irref_isk(nir->op2)) ||
nir->op1 == nir->op2)
J->cur.nins--; /* Drop useless PHIs. */
hiref2 = hisubst[ir->op2];
if (!((irref_isk(hiref) && irref_isk(hiref2)) || hiref == hiref2))
split_emit(J, IRTI(IR_PHI), hiref, hiref2);
break;
}
case IR_HIOP:
J->cur.nins--; /* Drop joining HIOP. */
ir->prev = nir->op1;
hi = nir->op2;
break;
default:
lua_assert(ir->o <= IR_NE); /* Comparisons. */
split_emit(J, IRTGI(IR_HIOP), hiref, hisubst[ir->op2]);
break;
}
} else
#endif
#if LJ_SOFTFP
if (ir->o == IR_SLOAD) {
if ((nir->op2 & IRSLOAD_CONVERT)) { /* Convert from number to int. */
nir->op2 &= ~IRSLOAD_CONVERT;
if (!(nir->op2 & IRSLOAD_TYPECHECK))
nir->t.irt = IRT_INT; /* Drop guard. */
split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), nref, nref);
ir->prev = split_num2int(J, nref, nref+1, irt_isguard(ir->t));
}
} else if (ir->o == IR_TOBIT) {
IRRef tmp, op1 = ir->op1;
J->cur.nins--;
#if LJ_LE
tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), oir[op1].prev, hisubst[op1]);
#else
tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), hisubst[op1], oir[op1].prev);
#endif
ir->prev = split_emit(J, IRTI(IR_CALLN), tmp, IRCALL_lj_vm_tobit);
} else if (ir->o == IR_TOSTR) {
if (hisubst[ir->op1]) {
if (irref_isk(ir->op1))
nir->op1 = ir->op1;
else
split_emit(J, IRT(IR_HIOP, IRT_NIL), hisubst[ir->op1], nref);
}
} else if (ir->o == IR_HREF || ir->o == IR_NEWREF) {
if (irref_isk(ir->op2) && hisubst[ir->op2])
nir->op2 = ir->op2;
} else
#endif
if (ir->o == IR_CONV) { /* See above, too. */
IRType st = (IRType)(ir->op2 & IRCONV_SRCMASK);
#if LJ_32 && LJ_HASFFI
if (st == IRT_I64 || st == IRT_U64) { /* Conversion from 64 bit int. */
#if LJ_SOFTFP
if (irt_isfloat(ir->t)) {
split_call_l(J, hisubst, oir, ir,
st == IRT_I64 ? IRCALL_fp64_l2f : IRCALL_fp64_ul2f);
J->cur.nins--; /* Drop unused HIOP. */
}
#else
if (irt_isfp(ir->t)) { /* 64 bit integer to FP conversion. */
ir->prev = split_emit(J, IRT(IR_HIOP, irt_type(ir->t)),
hisubst[ir->op1], nref);
}
#endif
else { /* Truncate to lower 32 bits. */
fwdlo:
ir->prev = nir->op1; /* Forward loword. */
/* Replace with NOP to avoid messing up the snapshot logic. */
nir->ot = IRT(IR_NOP, IRT_NIL);
nir->op1 = nir->op2 = 0;
}
}
#endif
#if LJ_SOFTFP && LJ_32 && LJ_HASFFI
else if (irt_isfloat(ir->t)) {
if (st == IRT_NUM) {
split_call_l(J, hisubst, oir, ir, IRCALL_softfp_d2f);
J->cur.nins--; /* Drop unused HIOP. */
} else {
nir->o = IR_CALLN;
nir->op2 = st == IRT_INT ? IRCALL_softfp_i2f : IRCALL_softfp_ui2f;
}
} else if (st == IRT_FLOAT) {
nir->o = IR_CALLN;
nir->op2 = irt_isint(ir->t) ? IRCALL_softfp_f2i : IRCALL_softfp_f2ui;
} else
#endif
#if LJ_SOFTFP
if (st == IRT_NUM || (LJ_32 && LJ_HASFFI && st == IRT_FLOAT)) {
if (irt_isguard(ir->t)) {
lua_assert(st == IRT_NUM && irt_isint(ir->t));
J->cur.nins--;
ir->prev = split_num2int(J, nir->op1, hisubst[ir->op1], 1);
} else {
split_call_l(J, hisubst, oir, ir,
#if LJ_32 && LJ_HASFFI
st == IRT_NUM ?
(irt_isint(ir->t) ? IRCALL_softfp_d2i : IRCALL_softfp_d2ui) :
(irt_isint(ir->t) ? IRCALL_softfp_f2i : IRCALL_softfp_f2ui)
#else
IRCALL_softfp_d2i
#endif
);
J->cur.nins--; /* Drop unused HIOP. */
}
}
#endif
} else if (ir->o == IR_CALLXS) {
IRRef hiref;
split_call:
hiref = hisubst[ir->op1];
if (hiref) {
IROpT ot = nir->ot;
IRRef op2 = nir->op2;
nir->ot = IRT(IR_CARG, IRT_NIL);
#if LJ_LE
nir->op2 = hiref;
#else
nir->op2 = nir->op1; nir->op1 = hiref;
#endif
ir->prev = nref = split_emit(J, ot, nref, op2);
}
if (LJ_SOFTFP ? irt_is64(ir->t) : irt_isint64(ir->t))
hi = split_emit(J,
IRT(IR_HIOP, (LJ_SOFTFP && irt_isnum(ir->t)) ? IRT_SOFTFP : IRT_INT),
nref, nref);
} else if (ir->o == IR_CARG) {
IRRef hiref = hisubst[ir->op1];
if (hiref) {
IRRef op2 = nir->op2;
#if LJ_LE
nir->op2 = hiref;
#else
nir->op2 = nir->op1; nir->op1 = hiref;
#endif
ir->prev = nref = split_emit(J, IRT(IR_CARG, IRT_NIL), nref, op2);
nir = IR(nref);
}
hiref = hisubst[ir->op2];
if (hiref) {
#if !LJ_TARGET_X86
int carg = 0;
IRIns *cir;
for (cir = IR(nir->op1); cir->o == IR_CARG; cir = IR(cir->op1))
carg++;
if ((carg & 1) == 0) { /* Align 64 bit arguments. */
IRRef op2 = nir->op2;
nir->op2 = REF_NIL;
nref = split_emit(J, IRT(IR_CARG, IRT_NIL), nref, op2);
nir = IR(nref);
}
#endif
#if LJ_BE
{ IRRef tmp = nir->op2; nir->op2 = hiref; hiref = tmp; }
#endif
ir->prev = split_emit(J, IRT(IR_CARG, IRT_NIL), nref, hiref);
}
} else if (ir->o == IR_CNEWI) {
if (hisubst[ir->op2])
split_emit(J, IRT(IR_HIOP, IRT_NIL), nref, hisubst[ir->op2]);
} else if (ir->o == IR_LOOP) {
J->loopref = nref; /* Needed by assembler. */
}
hisubst[ref] = hi; /* Store hiword substitution. */
}
if (snref == nins) { /* Substitution for last snapshot. */
snap->ref = J->cur.nins;
split_subst_snap(J, snap, oir);
}
/* Add PHI marks. */
for (ref = J->cur.nins-1; ref >= REF_FIRST; ref--) {
IRIns *ir = IR(ref);
if (ir->o != IR_PHI) break;
if (!irref_isk(ir->op1)) irt_setphi(IR(ir->op1)->t);
if (ir->op2 > J->loopref) irt_setphi(IR(ir->op2)->t);
}
}
