static TRef crec_tv_ct(jit_State *J, CType *s, CTypeID sid, TRef sp)
{
CTState *cts = ctype_ctsG(J2G(J));
CTInfo sinfo = s->info;
lua_assert(!ctype_isenum(sinfo));
if (ctype_isnum(sinfo)) {
IRType t = crec_ct2irt(s);
TRef tr;
if (t == IRT_CDATA)
goto err_nyi; /* NYI: copyval of >64 bit integers. */
tr = emitir(IRT(IR_XLOAD, t), sp, 0);
if (t == IRT_FLOAT || t == IRT_U32) { /* Keep uint32_t/float as numbers. */
return emitconv(tr, IRT_NUM, t, 0);
} else if (t == IRT_I64 || t == IRT_U64) { /* Box 64 bit integer. */
sp = tr;
lj_needsplit(J);
} else if ((sinfo & CTF_BOOL)) {
/* Assume not equal to zero. Fixup and emit pending guard later. */
lj_ir_set(J, IRTGI(IR_NE), tr, lj_ir_kint(J, 0));
J->postproc = LJ_POST_FIXGUARD;
return TREF_TRUE;
} else {
return tr;
}
} else if (ctype_isptr(sinfo)) {
IRType t = (LJ_64 && s->size == 8) ? IRT_P64 : IRT_P32;
sp = emitir(IRT(IR_XLOAD, t), sp, 0);
} else if (ctype_isrefarray(sinfo) || ctype_isstruct(sinfo)) {
cts->L = J->L;
sid = lj_ctype_intern(cts, CTINFO_REF(sid), CTSIZE_PTR); /* Create ref. */
} else if (ctype_iscomplex(sinfo)) { /* Unbox/box complex. */
IRType t = s->size == 2*sizeof(double) ? IRT_NUM : IRT_FLOAT;
ptrdiff_t esz = (ptrdiff_t)(s->size >> 1);
TRef ptr, tr1, tr2, dp;
dp = emitir(IRTG(IR_CNEW, IRT_CDATA), lj_ir_kint(J, sid), TREF_NIL);
tr1 = emitir(IRT(IR_XLOAD, t), sp, 0);
ptr = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, esz));
tr2 = emitir(IRT(IR_XLOAD, t), ptr, 0);
ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, sizeof(GCcdata)));
emitir(IRT(IR_XSTORE, t), ptr, tr1);
ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, sizeof(GCcdata)+esz));
emitir(IRT(IR_XSTORE, t), ptr, tr2);
return dp;
} else {
/* Replay snapshot state to setup side trace. */
void lj_snap_replay(jit_State *J, GCtrace *T)
{
SnapShot *snap = &T->snap[J->exitno];
SnapEntry *map = &T->snapmap[snap->mapofs];
MSize n, nent = snap->nent;
BloomFilter seen = 0;
int pass23 = 0;
J->framedepth = 0;
/* Emit IR for slots inherited from parent snapshot. */
for (n = 0; n < nent; n++) {
SnapEntry sn = map[n];
BCReg s = snap_slot(sn);
IRRef ref = snap_ref(sn);
IRIns *ir = &T->ir[ref];
TRef tr;
/* The bloom filter avoids O(nent^2) overhead for de-duping slots. */
if (bloomtest(seen, ref) && (tr = snap_dedup(J, map, n, ref)) != 0)
goto setslot;
bloomset(seen, ref);
if (irref_isk(ref)) {
tr = snap_replay_const(J, ir);
} else if (!regsp_used(ir->prev)) {
pass23 = 1;
lua_assert(s != 0);
tr = s;
} else {
IRType t = irt_type(ir->t);
uint32_t mode = IRSLOAD_INHERIT|IRSLOAD_PARENT;
if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) t = IRT_NUM;
if (ir->o == IR_SLOAD) mode |= (ir->op2 & IRSLOAD_READONLY);
tr = emitir_raw(IRT(IR_SLOAD, t), s, mode);
}
setslot:
J->slot[s] = tr | (sn&(SNAP_CONT|SNAP_FRAME)); /* Same as TREF_* flags. */
J->framedepth += ((sn & (SNAP_CONT|SNAP_FRAME)) && s);
if ((sn & SNAP_FRAME))
J->baseslot = s+1;
}
if (pass23) {
IRIns *irlast = &T->ir[snap->ref];
pass23 = 0;
/* Emit dependent PVALs. */
for (n = 0; n < nent; n++) {
SnapEntry sn = map[n];
IRRef refp = snap_ref(sn);
IRIns *ir = &T->ir[refp];
if (regsp_reg(ir->r) == RID_SUNK) {
if (J->slot[snap_slot(sn)] != snap_slot(sn)) continue;
pass23 = 1;
lua_assert(ir->o == IR_TNEW || ir->o == IR_TDUP ||
ir->o == IR_CNEW || ir->o == IR_CNEWI);
if (ir->op1 >= T->nk) snap_pref(J, T, map, nent, seen, ir->op1);
if (ir->op2 >= T->nk) snap_pref(J, T, map, nent, seen, ir->op2);
if (LJ_HASFFI && ir->o == IR_CNEWI) {
if (LJ_32 && refp+1 < T->nins && (ir+1)->o == IR_HIOP)
snap_pref(J, T, map, nent, seen, (ir+1)->op2);
} else {
IRIns *irs;
for (irs = ir+1; irs < irlast; irs++)
if (irs->r == RID_SINK && snap_sunk_store(T, ir, irs)) {
if (snap_pref(J, T, map, nent, seen, irs->op2) == 0)
snap_pref(J, T, map, nent, seen, T->ir[irs->op2].op1);
else if ((LJ_SOFTFP || (LJ_32 && LJ_HASFFI)) &&
irs+1 < irlast && (irs+1)->o == IR_HIOP)
snap_pref(J, T, map, nent, seen, (irs+1)->op2);
}
}
} else if (!irref_isk(refp) && !regsp_used(ir->prev)) {
lua_assert(ir->o == IR_CONV && ir->op2 == IRCONV_NUM_INT);
J->slot[snap_slot(sn)] = snap_pref(J, T, map, nent, seen, ir->op1);
}
}
/* Replay sunk instructions. */
for (n = 0; pass23 && n < nent; n++) {
SnapEntry sn = map[n];
IRRef refp = snap_ref(sn);
IRIns *ir = &T->ir[refp];
if (regsp_reg(ir->r) == RID_SUNK) {
TRef op1, op2;
if (J->slot[snap_slot(sn)] != snap_slot(sn)) { /* De-dup allocs. */
J->slot[snap_slot(sn)] = J->slot[J->slot[snap_slot(sn)]];
continue;
}
op1 = ir->op1;
if (op1 >= T->nk) op1 = snap_pref(J, T, map, nent, seen, op1);
op2 = ir->op2;
if (op2 >= T->nk) op2 = snap_pref(J, T, map, nent, seen, op2);
if (LJ_HASFFI && ir->o == IR_CNEWI) {
if (LJ_32 && refp+1 < T->nins && (ir+1)->o == IR_HIOP) {
lj_needsplit(J); /* Emit joining HIOP. */
op2 = emitir_raw(IRT(IR_HIOP, IRT_I64), op2,
snap_pref(J, T, map, nent, seen, (ir+1)->op2));
}
J->slot[snap_slot(sn)] = emitir(ir->ot & ~(IRT_MARK|IRT_ISPHI), op1, op2);
} else {
IRIns *irs;
TRef tr = emitir(ir->ot, op1, op2);
J->slot[snap_slot(sn)] = tr;
for (irs = ir+1; irs < irlast; irs++)
if (irs->r == RID_SINK && snap_sunk_store(T, ir, irs)) {
IRIns *irr = &T->ir[irs->op1];
TRef val, key = irr->op2, tmp = tr;
if (irr->o != IR_FREF) {
IRIns *irk = &T->ir[key];
if (irr->o == IR_HREFK)
key = lj_ir_kslot(J, snap_replay_const(J, &T->ir[irk->op1]),
irk->op2);
else
key = snap_replay_const(J, irk);
if (irr->o == IR_HREFK || irr->o == IR_AREF) {
IRIns *irf = &T->ir[irr->op1];
tmp = emitir(irf->ot, tmp, irf->op2);
}
}
tmp = emitir(irr->ot, tmp, key);
val = snap_pref(J, T, map, nent, seen, irs->op2);
if (val == 0) {
IRIns *irc = &T->ir[irs->op2];
lua_assert(irc->o == IR_CONV && irc->op2 == IRCONV_NUM_INT);
val = snap_pref(J, T, map, nent, seen, irc->op1);
val = emitir(IRTN(IR_CONV), val, IRCONV_NUM_INT);
} else if ((LJ_SOFTFP || (LJ_32 && LJ_HASFFI)) &&
irs+1 < irlast && (irs+1)->o == IR_HIOP) {
IRType t = IRT_I64;
if (LJ_SOFTFP && irt_type((irs+1)->t) == IRT_SOFTFP)
t = IRT_NUM;
lj_needsplit(J);
if (irref_isk(irs->op2) && irref_isk((irs+1)->op2)) {
uint64_t k = (uint32_t)T->ir[irs->op2].i +
((uint64_t)T->ir[(irs+1)->op2].i << 32);
val = lj_ir_k64(J, t == IRT_I64 ? IR_KINT64 : IR_KNUM,
lj_ir_k64_find(J, k));
} else {
val = emitir_raw(IRT(IR_HIOP, t), val,
snap_pref(J, T, map, nent, seen, (irs+1)->op2));
}
tmp = emitir(IRT(irs->o, t), tmp, val);
continue;
}
tmp = emitir(irs->ot, tmp, val);
} else if (LJ_HASFFI && irs->o == IR_XBAR && ir->o == IR_CNEW) {
emitir(IRT(IR_XBAR, IRT_NIL), 0, 0);
}
}
}
}
}
J->base = J->slot + J->baseslot;
J->maxslot = snap->nslots - J->baseslot;
lj_snap_add(J);
if (pass23) /* Need explicit GC step _after_ initial snapshot. */
emitir_raw(IRTG(IR_GCSTEP, IRT_NIL), 0, 0);
}
static TRef crec_ct_ct(jit_State *J, CType *d, CType *s, TRef dp, TRef sp,
void *svisnz)
{
CTSize dsize = d->size, ssize = s->size;
CTInfo dinfo = d->info, sinfo = s->info;
IRType dt = crec_ct2irt(d);
IRType st = crec_ct2irt(s);
if (ctype_type(dinfo) > CT_MAYCONVERT || ctype_type(sinfo) > CT_MAYCONVERT)
goto err_conv;
/*
** Note: Unlike lj_cconv_ct_ct(), sp holds the _value_ of pointers and
** numbers up to 8 bytes. Otherwise sp holds a pointer.
*/
switch (cconv_idx2(dinfo, sinfo)) {
/* Destination is a bool. */
case CCX(B, B):
goto xstore; /* Source operand is already normalized. */
case CCX(B, I):
case CCX(B, F):
if (st != IRT_CDATA) {
/* Specialize to the result of a comparison against 0. */
TRef zero = (st == IRT_NUM || st == IRT_FLOAT) ? lj_ir_knum(J, 0) :
(st == IRT_I64 || st == IRT_U64) ? lj_ir_kint64(J, 0) :
lj_ir_kint(J, 0);
int isnz = crec_isnonzero(s, svisnz);
emitir(IRTG(isnz ? IR_NE : IR_EQ, st), sp, zero);
sp = lj_ir_kint(J, isnz);
goto xstore;
}
goto err_nyi;
/* Destination is an integer. */
case CCX(I, B):
case CCX(I, I):
conv_I_I:
if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
/* Extend 32 to 64 bit integer. */
if (dsize == 8 && ssize < 8 && !(LJ_64 && (sinfo & CTF_UNSIGNED)))
sp = emitconv(sp, dt, ssize < 4 ? IRT_INT : st,
(sinfo & CTF_UNSIGNED) ? 0 : IRCONV_SEXT);
else if (dsize < 8 && ssize == 8) /* Truncate from 64 bit integer. */
sp = emitconv(sp, dsize < 4 ? IRT_INT : dt, st, 0);
else if (st == IRT_INT)
sp = lj_opt_narrow_toint(J, sp);
xstore:
if (dt == IRT_I64 || dt == IRT_U64) lj_needsplit(J);
if (dp == 0) return sp;
emitir(IRT(IR_XSTORE, dt), dp, sp);
break;
case CCX(I, C):
sp = emitir(IRT(IR_XLOAD, st), sp, 0); /* Load re. */
/* fallthrough */
case CCX(I, F):
if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
sp = emitconv(sp, dsize < 4 ? IRT_INT : dt, st, IRCONV_TRUNC|IRCONV_ANY);
goto xstore;
case CCX(I, P):
case CCX(I, A):
sinfo = CTINFO(CT_NUM, CTF_UNSIGNED);
ssize = CTSIZE_PTR;
st = IRT_UINTP;
goto conv_I_I;
/* Destination is a floating-point number. */
case CCX(F, B):
case CCX(F, I):
conv_F_I:
if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
sp = emitconv(sp, dt, ssize < 4 ? IRT_INT : st, 0);
goto xstore;
case CCX(F, C):
sp = emitir(IRT(IR_XLOAD, st), sp, 0); /* Load re. */
/* fallthrough */
case CCX(F, F):
conv_F_F:
if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
if (dt != st) sp = emitconv(sp, dt, st, 0);
goto xstore;
/* Destination is a complex number. */
case CCX(C, I):
case CCX(C, F):
{ /* Clear im. */
TRef ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, (dsize >> 1)));
emitir(IRT(IR_XSTORE, dt), ptr, lj_ir_knum(J, 0));
}
/* Convert to re. */
if ((sinfo & CTF_FP)) goto conv_F_F; else goto conv_F_I;
case CCX(C, C):
if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
{
TRef re, im, ptr;
re = emitir(IRT(IR_XLOAD, st), sp, 0);
ptr = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, (ssize >> 1)));
im = emitir(IRT(IR_XLOAD, st), ptr, 0);
if (dt != st) {
re = emitconv(re, dt, st, 0);
im = emitconv(im, dt, st, 0);
}
emitir(IRT(IR_XSTORE, dt), dp, re);
ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, (dsize >> 1)));
emitir(IRT(IR_XSTORE, dt), ptr, im);
}
break;
/* Destination is a vector. */
case CCX(V, I):
case CCX(V, F):
case CCX(V, C):
case CCX(V, V):
goto err_nyi;
/* Destination is a pointer. */
case CCX(P, P):
case CCX(P, A):
case CCX(P, S):
/* There are only 32 bit pointers/addresses on 32 bit machines.
** Also ok on x64, since all 32 bit ops clear the upper part of the reg.
*/
goto xstore;
case CCX(P, I):
if (st == IRT_CDATA) goto err_nyi;
if (!LJ_64 && ssize == 8) /* Truncate from 64 bit integer. */
sp = emitconv(sp, IRT_U32, st, 0);
goto xstore;
case CCX(P, F):
if (st == IRT_CDATA) goto err_nyi;
/* The signed conversion is cheaper. x64 really has 47 bit pointers. */
sp = emitconv(sp, (LJ_64 && dsize == 8) ? IRT_I64 : IRT_U32,
st, IRCONV_TRUNC|IRCONV_ANY);
goto xstore;
/* Destination is an array. */
case CCX(A, A):
goto err_nyi;
/* Destination is a struct/union. */
case CCX(S, S):
goto err_nyi;
default:
err_conv:
err_nyi:
lj_trace_err(J, LJ_TRERR_NYICONV);
break;
}
return 0;
}