/* Restore a value from the trace exit state. */
static void snap_restoreval(jit_State *J, GCtrace *T, ExitState *ex,
SnapNo snapno, BloomFilter rfilt,
IRRef ref, TValue *o)
{
IRIns *ir = &T->ir[ref];
IRType1 t = ir->t;
RegSP rs = ir->prev;
if (irref_isk(ref)) { /* Restore constant slot. */
lj_ir_kvalue(J->L, o, ir);
return;
}
if (LJ_UNLIKELY(bloomtest(rfilt, ref)))
rs = snap_renameref(T, snapno, ref, rs);
lua_assert(!LJ_GC64); /* TODO_GC64: handle 64 bit references. */
if (ra_hasspill(regsp_spill(rs))) { /* Restore from spill slot. */
int32_t *sps = &ex->spill[regsp_spill(rs)];
if (irt_isinteger(t)) {
setintV(o, *sps);
#if !LJ_SOFTFP
} else if (irt_isnum(t)) {
o->u64 = *(uint64_t *)sps;
#endif
} else if (LJ_64 && irt_islightud(t)) {
/* 64 bit lightuserdata which may escape already has the tag bits. */
o->u64 = *(uint64_t *)sps;
} else {
lua_assert(!irt_ispri(t)); /* PRI refs never have a spill slot. */
setgcV(J->L, o, (GCobj *)(uintptr_t)*(GCSize *)sps, irt_toitype(t));
}
} else { /* Restore from register. */
Reg r = regsp_reg(rs);
if (ra_noreg(r)) {
lua_assert(ir->o == IR_CONV && ir->op2 == IRCONV_NUM_INT);
snap_restoreval(J, T, ex, snapno, rfilt, ir->op1, o);
if (LJ_DUALNUM) setnumV(o, (lua_Number)intV(o));
return;
} else if (irt_isinteger(t)) {
setintV(o, (int32_t)ex->gpr[r-RID_MIN_GPR]);
#if !LJ_SOFTFP
} else if (irt_isnum(t)) {
setnumV(o, ex->fpr[r-RID_MIN_FPR]);
#endif
} else if (LJ_64 && irt_is64(t)) {
/* 64 bit values that already have the tag bits. */
o->u64 = ex->gpr[r-RID_MIN_GPR];
} else if (irt_ispri(t)) {
setpriV(o, irt_toitype(t));
} else {
setgcV(J->L, o, (GCobj *)ex->gpr[r-RID_MIN_GPR], irt_toitype(t));
}
}
}
/* Unsink allocation from the trace exit state. Unsink sunk stores. */
static void snap_unsink(jit_State *J, GCtrace *T, ExitState *ex,
SnapNo snapno, BloomFilter rfilt,
IRIns *ir, TValue *o)
{
lua_assert(ir->o == IR_TNEW || ir->o == IR_TDUP ||
ir->o == IR_CNEW || ir->o == IR_CNEWI);
#if LJ_HASFFI
if (ir->o == IR_CNEW || ir->o == IR_CNEWI) {
CTState *cts = ctype_cts(J->L);
CTypeID id = (CTypeID)T->ir[ir->op1].i;
CTSize sz;
CTInfo info = lj_ctype_info(cts, id, &sz);
GCcdata *cd = lj_cdata_newx(cts, id, sz, info);
setcdataV(J->L, o, cd);
if (ir->o == IR_CNEWI) {
uint8_t *p = (uint8_t *)cdataptr(cd);
lua_assert(sz == 4 || sz == 8);
if (LJ_32 && sz == 8 && ir+1 < T->ir + T->nins && (ir+1)->o == IR_HIOP) {
snap_restoredata(T, ex, snapno, rfilt, (ir+1)->op2, LJ_LE?p+4:p, 4);
if (LJ_BE) p += 4;
sz = 4;
}
snap_restoredata(T, ex, snapno, rfilt, ir->op2, p, sz);
} else {
IRIns *irs, *irlast = &T->ir[T->snap[snapno].ref];
for (irs = ir+1; irs < irlast; irs++)
if (irs->r == RID_SINK && snap_sunk_store(T, ir, irs)) {
IRIns *iro = &T->ir[T->ir[irs->op1].op2];
uint8_t *p = (uint8_t *)cd;
CTSize szs;
lua_assert(irs->o == IR_XSTORE && T->ir[irs->op1].o == IR_ADD);
lua_assert(iro->o == IR_KINT || iro->o == IR_KINT64);
if (irt_is64(irs->t)) szs = 8;
else if (irt_isi8(irs->t) || irt_isu8(irs->t)) szs = 1;
else if (irt_isi16(irs->t) || irt_isu16(irs->t)) szs = 2;
else szs = 4;
if (LJ_64 && iro->o == IR_KINT64)
p += (int64_t)ir_k64(iro)->u64;
else
p += iro->i;
lua_assert(p >= (uint8_t *)cdataptr(cd) &&
p + szs <= (uint8_t *)cdataptr(cd) + sz);
if (LJ_32 && irs+1 < T->ir + T->nins && (irs+1)->o == IR_HIOP) {
lua_assert(szs == 4);
snap_restoredata(T, ex, snapno, rfilt, (irs+1)->op2, LJ_LE?p+4:p,4);
if (LJ_BE) p += 4;
}
snap_restoredata(T, ex, snapno, rfilt, irs->op2, p, szs);
}
}
} else
#endif
{
IRIns *irs, *irlast;
GCtab *t = ir->o == IR_TNEW ? lj_tab_new(J->L, ir->op1, ir->op2) :
lj_tab_dup(J->L, ir_ktab(&T->ir[ir->op1]));
settabV(J->L, o, t);
irlast = &T->ir[T->snap[snapno].ref];
for (irs = ir+1; irs < irlast; irs++)
if (irs->r == RID_SINK && snap_sunk_store(T, ir, irs)) {
IRIns *irk = &T->ir[irs->op1];
TValue tmp, *val;
lua_assert(irs->o == IR_ASTORE || irs->o == IR_HSTORE ||
irs->o == IR_FSTORE);
if (irk->o == IR_FREF) {
lua_assert(irk->op2 == IRFL_TAB_META);
snap_restoreval(J, T, ex, snapno, rfilt, irs->op2, &tmp);
/* NOBARRIER: The table is new (marked white). */
setgcref(t->metatable, obj2gco(tabV(&tmp)));
} else {
irk = &T->ir[irk->op2];
if (irk->o == IR_KSLOT) irk = &T->ir[irk->op1];
lj_ir_kvalue(J->L, &tmp, irk);
val = lj_tab_set(J->L, t, &tmp);
/* NOBARRIER: The table is new (marked white). */
snap_restoreval(J, T, ex, snapno, rfilt, irs->op2, val);
if (LJ_SOFTFP && irs+1 < T->ir + T->nins && (irs+1)->o == IR_HIOP) {
snap_restoreval(J, T, ex, snapno, rfilt, (irs+1)->op2, &tmp);
val->u32.hi = tmp.u32.lo;
}
}
}
}
}
/* Restore interpreter state from exit state with the help of a snapshot. */
const BCIns *lj_snap_restore(jit_State *J, void *exptr)
{
ExitState *ex = (ExitState *)exptr;
SnapNo snapno = J->exitno; /* For now, snapno == exitno. */
GCtrace *T = traceref(J, J->parent);
SnapShot *snap = &T->snap[snapno];
MSize n, nent = snap->nent;
SnapEntry *map = &T->snapmap[snap->mapofs];
SnapEntry *flinks = map + nent + snap->depth;
int32_t ftsz0;
BCReg nslots = snap->nslots;
TValue *frame;
BloomFilter rfilt = snap_renamefilter(T, snapno);
const BCIns *pc = snap_pc(map[nent]);
lua_State *L = J->L;
/* Set interpreter PC to the next PC to get correct error messages. */
setcframe_pc(cframe_raw(L->cframe), pc+1);
/* Make sure the stack is big enough for the slots from the snapshot. */
if (LJ_UNLIKELY(L->base + nslots > tvref(L->maxstack))) {
L->top = curr_topL(L);
lj_state_growstack(L, nslots - curr_proto(L)->framesize);
}
/* Fill stack slots with data from the registers and spill slots. */
frame = L->base-1;
ftsz0 = frame_ftsz(frame); /* Preserve link to previous frame in slot #0. */
for (n = 0; n < nent; n++) {
SnapEntry sn = map[n];
IRRef ref = snap_ref(sn);
BCReg s = snap_slot(sn);
TValue *o = &frame[s]; /* Stack slots are relative to start frame. */
IRIns *ir = &T->ir[ref];
if (irref_isk(ref)) { /* Restore constant slot. */
lj_ir_kvalue(L, o, ir);
if ((sn & (SNAP_CONT|SNAP_FRAME))) {
/* Overwrite tag with frame link. */
o->fr.tp.ftsz = s != 0 ? (int32_t)*flinks-- : ftsz0;
if ((sn & SNAP_FRAME)) {
GCfunc *fn = ir_kfunc(ir);
if (isluafunc(fn)) {
MSize framesize = funcproto(fn)->framesize;
L->base = ++o;
if (LJ_UNLIKELY(o + framesize > tvref(L->maxstack))) {
ptrdiff_t fsave = savestack(L, frame);
L->top = o;
lj_state_growstack(L, framesize); /* Grow again. */
frame = restorestack(L, fsave);
}
}
}
}
} else if (!(sn & SNAP_NORESTORE)) {
IRType1 t = ir->t;
RegSP rs = ir->prev;
lua_assert(!(sn & (SNAP_CONT|SNAP_FRAME)));
if (LJ_UNLIKELY(bloomtest(rfilt, ref)))
rs = snap_renameref(T, snapno, ref, rs);
if (ra_hasspill(regsp_spill(rs))) { /* Restore from spill slot. */
int32_t *sps = &ex->spill[regsp_spill(rs)];
if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) {
o->u32.lo = (uint32_t)*sps;
} else if (irt_isinteger(t)) {
setintV(o, *sps);
#if !LJ_SOFTFP
} else if (irt_isnum(t)) {
o->u64 = *(uint64_t *)sps;
#endif
#if LJ_64
} else if (irt_islightud(t)) {
/* 64 bit lightuserdata which may escape already has the tag bits. */
o->u64 = *(uint64_t *)sps;
#endif
} else {
lua_assert(!irt_ispri(t)); /* PRI refs never have a spill slot. */
setgcrefi(o->gcr, *sps);
setitype(o, irt_toitype(t));
}
} else { /* Restore from register. */
Reg r = regsp_reg(rs);
lua_assert(ra_hasreg(r));
if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) {
o->u32.lo = (uint32_t)ex->gpr[r-RID_MIN_GPR];
} else if (irt_isinteger(t)) {
setintV(o, (int32_t)ex->gpr[r-RID_MIN_GPR]);
#if !LJ_SOFTFP
} else if (irt_isnum(t)) {
setnumV(o, ex->fpr[r-RID_MIN_FPR]);
#endif
#if LJ_64
} else if (irt_islightud(t)) {
/* 64 bit lightuserdata which may escape already has the tag bits. */
o->u64 = ex->gpr[r-RID_MIN_GPR];
#endif
} else {
if (!irt_ispri(t))
setgcrefi(o->gcr, ex->gpr[r-RID_MIN_GPR]);
setitype(o, irt_toitype(t));
}
}
if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) {
rs = (ir+1)->prev;
if (LJ_UNLIKELY(bloomtest(rfilt, ref+1)))
rs = snap_renameref(T, snapno, ref+1, rs);
o->u32.hi = (ra_hasspill(regsp_spill(rs))) ?
(uint32_t)*&ex->spill[regsp_spill(rs)] :
(uint32_t)ex->gpr[regsp_reg(rs)-RID_MIN_GPR];
}
}
}
switch (bc_op(*pc)) {
case BC_CALLM: case BC_CALLMT: case BC_RETM: case BC_TSETM:
L->top = frame + nslots;
break;
default:
L->top = curr_topL(L);
break;
}
lua_assert(map + nent == flinks);
return pc;
}
/* Array and hash load forwarding. */
static TRef fwd_ahload(jit_State *J, IRRef xref)
{
IRIns *xr = IR(xref);
IRRef lim = xref; /* Search limit. */
IRRef ref;
/* Search for conflicting stores. */
ref = J->chain[fins->o+IRDELTA_L2S];
while (ref > xref) {
IRIns *store = IR(ref);
switch (aa_ahref(J, xr, IR(store->op1))) {
case ALIAS_NO: break; /* Continue searching. */
case ALIAS_MAY: lim = ref; goto cselim; /* Limit search for load. */
case ALIAS_MUST: return store->op2; /* Store forwarding. */
}
ref = store->prev;
}
/* No conflicting store (yet): const-fold loads from allocations. */
{
IRIns *ir = (xr->o == IR_HREFK || xr->o == IR_AREF) ? IR(xr->op1) : xr;
IRRef tab = ir->op1;
ir = IR(tab);
if (ir->o == IR_TNEW || (ir->o == IR_TDUP && irref_isk(xr->op2))) {
/* A NEWREF with a number key may end up pointing to the array part.
** But it's referenced from HSTORE and not found in the ASTORE chain.
** For now simply consider this a conflict without forwarding anything.
*/
if (xr->o == IR_AREF) {
IRRef ref2 = J->chain[IR_NEWREF];
while (ref2 > tab) {
IRIns *newref = IR(ref2);
if (irt_isnum(IR(newref->op2)->t))
goto cselim;
ref2 = newref->prev;
}
}
/* NEWREF inhibits CSE for HREF, and dependent FLOADs from HREFK/AREF.
** But the above search for conflicting stores was limited by xref.
** So continue searching, limited by the TNEW/TDUP. Store forwarding
** is ok, too. A conflict does NOT limit the search for a matching load.
*/
while (ref > tab) {
IRIns *store = IR(ref);
switch (aa_ahref(J, xr, IR(store->op1))) {
case ALIAS_NO: break; /* Continue searching. */
case ALIAS_MAY: goto cselim; /* Conflicting store. */
case ALIAS_MUST: return store->op2; /* Store forwarding. */
}
ref = store->prev;
}
lua_assert(ir->o != IR_TNEW || irt_isnil(fins->t));
if (irt_ispri(fins->t)) {
return TREF_PRI(irt_type(fins->t));
} else if (irt_isnum(fins->t) || (LJ_DUALNUM && irt_isint(fins->t)) ||
irt_isstr(fins->t)) {
TValue keyv;
cTValue *tv;
IRIns *key = IR(xr->op2);
if (key->o == IR_KSLOT) key = IR(key->op1);
lj_ir_kvalue(J->L, &keyv, key);
tv = lj_tab_get(J->L, ir_ktab(IR(ir->op1)), &keyv);
lua_assert(itype2irt(tv) == irt_type(fins->t));
if (irt_isnum(fins->t))
return lj_ir_knum_u64(J, tv->u64);
else if (LJ_DUALNUM && irt_isint(fins->t))
return lj_ir_kint(J, intV(tv));
else
return lj_ir_kstr(J, strV(tv));
}
/* Othwerwise: don't intern as a constant. */
}
}
cselim:
/* Try to find a matching load. Below the conflicting store, if any. */
ref = J->chain[fins->o];
while (ref > lim) {
IRIns *load = IR(ref);
if (load->op1 == xref)
return ref; /* Load forwarding. */
ref = load->prev;
}
return 0; /* Conflict or no match. */
}
/* Restore interpreter state from exit state with the help of a snapshot. */
void lj_snap_restore(jit_State *J, void *exptr)
{
ExitState *ex = (ExitState *)exptr;
SnapNo snapno = J->exitno; /* For now, snapno == exitno. */
Trace *T = J->trace[J->parent];
SnapShot *snap = &T->snap[snapno];
BCReg s, nslots = snap->nslots;
IRRef2 *map = &T->snapmap[snap->mapofs];
IRRef2 *flinks = map + nslots + snap->nframelinks;
TValue *o, *newbase, *ntop;
BloomFilter rfilt = snap_renamefilter(T, snapno);
lua_State *L = J->L;
/* Make sure the stack is big enough for the slots from the snapshot. */
if (L->base + nslots >= L->maxstack) {
L->top = curr_topL(L);
lj_state_growstack(L, nslots - curr_proto(L)->framesize);
}
/* Fill stack slots with data from the registers and spill slots. */
newbase = NULL;
ntop = L->base;
for (s = 0, o = L->base-1; s < nslots; s++, o++) {
IRRef ref = snap_ref(map[s]);
if (ref) {
IRIns *ir = &T->ir[ref];
if (irref_isk(ref)) { /* Restore constant slot. */
lj_ir_kvalue(L, o, ir);
} else {
IRType1 t = ir->t;
RegSP rs = ir->prev;
if (LJ_UNLIKELY(bloomtest(rfilt, ref)))
rs = snap_renameref(T, snapno, ref, rs);
if (ra_hasspill(regsp_spill(rs))) { /* Restore from spill slot. */
int32_t *sps = &ex->spill[regsp_spill(rs)];
if (irt_isinteger(t)) {
setintV(o, *sps);
} else if (irt_isnum(t)) {
o->u64 = *(uint64_t *)sps;
} else {
lua_assert(!irt_ispri(t)); /* PRI refs never have a spill slot. */
setgcrefi(o->gcr, *sps);
setitype(o, irt_toitype(t));
}
} else if (ra_hasreg(regsp_reg(rs))) { /* Restore from register. */
Reg r = regsp_reg(rs);
if (irt_isinteger(t)) {
setintV(o, ex->gpr[r-RID_MIN_GPR]);
} else if (irt_isnum(t)) {
setnumV(o, ex->fpr[r-RID_MIN_FPR]);
} else {
if (!irt_ispri(t))
setgcrefi(o->gcr, ex->gpr[r-RID_MIN_GPR]);
setitype(o, irt_toitype(t));
}
} else { /* Restore frame slot. */
lua_assert(ir->o == IR_FRAME);
/* This works for both PTR and FUNC IR_FRAME. */
setgcrefp(o->fr.func, mref(T->ir[ir->op2].ptr, void));
if (s != 0) /* Do not overwrite link to previous frame. */
o->fr.tp.ftsz = (int32_t)*--flinks;
if (irt_isfunc(ir->t)) {
GCfunc *fn = gco2func(gcref(T->ir[ir->op2].gcr));
if (isluafunc(fn)) {
TValue *fs;
newbase = o+1;
fs = newbase + funcproto(fn)->framesize;
if (fs > ntop) ntop = fs; /* Update top for newly added frames. */
}
}
}
}
} else if (newbase) {
setnilV(o); /* Clear unreferenced slots of newly added frames. */
}
}
