/* Restore raw data from the trace exit state. */
static void snap_restoredata(GCtrace *T, ExitState *ex,
SnapNo snapno, BloomFilter rfilt,
IRRef ref, void *dst, CTSize sz)
{
IRIns *ir = &T->ir[ref];
RegSP rs = ir->prev;
int32_t *src;
uint64_t tmp;
if (irref_isk(ref)) {
if (ir->o == IR_KNUM || ir->o == IR_KINT64) {
src = mref(ir->ptr, int32_t);
} else if (sz == 8) {
tmp = (uint64_t)(uint32_t)ir->i;
src = (int32_t *)&tmp;
} else {
src = &ir->i;
}
} else {
if (LJ_UNLIKELY(bloomtest(rfilt, ref)))
rs = snap_renameref(T, snapno, ref, rs);
if (ra_hasspill(regsp_spill(rs))) {
src = &ex->spill[regsp_spill(rs)];
if (sz == 8 && !irt_is64(ir->t)) {
tmp = (uint64_t)(uint32_t)*src;
src = (int32_t *)&tmp;
}
} else {
Reg r = regsp_reg(rs);
if (ra_noreg(r)) {
/* Note: this assumes CNEWI is never used for SOFTFP split numbers. */
lua_assert(sz == 8 && ir->o == IR_CONV && ir->op2 == IRCONV_NUM_INT);
snap_restoredata(T, ex, snapno, rfilt, ir->op1, dst, 4);
*(lua_Number *)dst = (lua_Number)*(int32_t *)dst;
return;
}
src = (int32_t *)&ex->gpr[r-RID_MIN_GPR];
#if !LJ_SOFTFP
if (r >= RID_MAX_GPR) {
src = (int32_t *)&ex->fpr[r-RID_MIN_FPR];
#if LJ_TARGET_PPC
if (sz == 4) { /* PPC FPRs are always doubles. */
*(float *)dst = (float)*(double *)src;
return;
}
#else
if (LJ_BE && sz == 4) src++;
#endif
}
#endif
}
}
lua_assert(sz == 1 || sz == 2 || sz == 4 || sz == 8);
if (sz == 4) *(int32_t *)dst = *src;
else if (sz == 8) *(int64_t *)dst = *(int64_t *)src;
else if (sz == 1) *(int8_t *)dst = (int8_t)*src;
else *(int16_t *)dst = (int16_t)*src;
}
/* Restore a value from the trace exit state. */
static void snap_restoreval(jit_State *J, GCtrace *T, ExitState *ex,
SnapNo snapno, BloomFilter rfilt,
IRRef ref, TValue *o)
{
IRIns *ir = &T->ir[ref];
IRType1 t = ir->t;
RegSP rs = ir->prev;
if (irref_isk(ref)) { /* Restore constant slot. */
lj_ir_kvalue(J->L, o, ir);
return;
}
if (LJ_UNLIKELY(bloomtest(rfilt, ref)))
rs = snap_renameref(T, snapno, ref, rs);
lua_assert(!LJ_GC64); /* TODO_GC64: handle 64 bit references. */
if (ra_hasspill(regsp_spill(rs))) { /* Restore from spill slot. */
int32_t *sps = &ex->spill[regsp_spill(rs)];
if (irt_isinteger(t)) {
setintV(o, *sps);
#if !LJ_SOFTFP
} else if (irt_isnum(t)) {
o->u64 = *(uint64_t *)sps;
#endif
} else if (LJ_64 && irt_islightud(t)) {
/* 64 bit lightuserdata which may escape already has the tag bits. */
o->u64 = *(uint64_t *)sps;
} else {
lua_assert(!irt_ispri(t)); /* PRI refs never have a spill slot. */
setgcV(J->L, o, (GCobj *)(uintptr_t)*(GCSize *)sps, irt_toitype(t));
}
} else { /* Restore from register. */
Reg r = regsp_reg(rs);
if (ra_noreg(r)) {
lua_assert(ir->o == IR_CONV && ir->op2 == IRCONV_NUM_INT);
snap_restoreval(J, T, ex, snapno, rfilt, ir->op1, o);
if (LJ_DUALNUM) setnumV(o, (lua_Number)intV(o));
return;
} else if (irt_isinteger(t)) {
setintV(o, (int32_t)ex->gpr[r-RID_MIN_GPR]);
#if !LJ_SOFTFP
} else if (irt_isnum(t)) {
setnumV(o, ex->fpr[r-RID_MIN_FPR]);
#endif
} else if (LJ_64 && irt_is64(t)) {
/* 64 bit values that already have the tag bits. */
o->u64 = ex->gpr[r-RID_MIN_GPR];
} else if (irt_ispri(t)) {
setpriV(o, irt_toitype(t));
} else {
setgcV(J->L, o, (GCobj *)ex->gpr[r-RID_MIN_GPR], irt_toitype(t));
}
}
}
/* Restore interpreter state from exit state with the help of a snapshot. */
const BCIns *lj_snap_restore(jit_State *J, void *exptr)
{
ExitState *ex = (ExitState *)exptr;
SnapNo snapno = J->exitno; /* For now, snapno == exitno. */
GCtrace *T = traceref(J, J->parent);
SnapShot *snap = &T->snap[snapno];
MSize n, nent = snap->nent;
SnapEntry *map = &T->snapmap[snap->mapofs];
SnapEntry *flinks = map + nent + snap->depth;
int32_t ftsz0;
BCReg nslots = snap->nslots;
TValue *frame;
BloomFilter rfilt = snap_renamefilter(T, snapno);
const BCIns *pc = snap_pc(map[nent]);
lua_State *L = J->L;
/* Set interpreter PC to the next PC to get correct error messages. */
setcframe_pc(cframe_raw(L->cframe), pc+1);
/* Make sure the stack is big enough for the slots from the snapshot. */
if (LJ_UNLIKELY(L->base + nslots > tvref(L->maxstack))) {
L->top = curr_topL(L);
lj_state_growstack(L, nslots - curr_proto(L)->framesize);
}
/* Fill stack slots with data from the registers and spill slots. */
frame = L->base-1;
ftsz0 = frame_ftsz(frame); /* Preserve link to previous frame in slot #0. */
for (n = 0; n < nent; n++) {
SnapEntry sn = map[n];
IRRef ref = snap_ref(sn);
BCReg s = snap_slot(sn);
TValue *o = &frame[s]; /* Stack slots are relative to start frame. */
IRIns *ir = &T->ir[ref];
if (irref_isk(ref)) { /* Restore constant slot. */
lj_ir_kvalue(L, o, ir);
if ((sn & (SNAP_CONT|SNAP_FRAME))) {
/* Overwrite tag with frame link. */
o->fr.tp.ftsz = s != 0 ? (int32_t)*flinks-- : ftsz0;
if ((sn & SNAP_FRAME)) {
GCfunc *fn = ir_kfunc(ir);
if (isluafunc(fn)) {
MSize framesize = funcproto(fn)->framesize;
L->base = ++o;
if (LJ_UNLIKELY(o + framesize > tvref(L->maxstack))) {
ptrdiff_t fsave = savestack(L, frame);
L->top = o;
lj_state_growstack(L, framesize); /* Grow again. */
frame = restorestack(L, fsave);
}
}
}
}
} else if (!(sn & SNAP_NORESTORE)) {
IRType1 t = ir->t;
RegSP rs = ir->prev;
lua_assert(!(sn & (SNAP_CONT|SNAP_FRAME)));
if (LJ_UNLIKELY(bloomtest(rfilt, ref)))
rs = snap_renameref(T, snapno, ref, rs);
if (ra_hasspill(regsp_spill(rs))) { /* Restore from spill slot. */
int32_t *sps = &ex->spill[regsp_spill(rs)];
if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) {
o->u32.lo = (uint32_t)*sps;
} else if (irt_isinteger(t)) {
setintV(o, *sps);
#if !LJ_SOFTFP
} else if (irt_isnum(t)) {
o->u64 = *(uint64_t *)sps;
#endif
#if LJ_64
} else if (irt_islightud(t)) {
/* 64 bit lightuserdata which may escape already has the tag bits. */
o->u64 = *(uint64_t *)sps;
#endif
} else {
lua_assert(!irt_ispri(t)); /* PRI refs never have a spill slot. */
setgcrefi(o->gcr, *sps);
setitype(o, irt_toitype(t));
}
} else { /* Restore from register. */
Reg r = regsp_reg(rs);
lua_assert(ra_hasreg(r));
if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) {
o->u32.lo = (uint32_t)ex->gpr[r-RID_MIN_GPR];
} else if (irt_isinteger(t)) {
setintV(o, (int32_t)ex->gpr[r-RID_MIN_GPR]);
#if !LJ_SOFTFP
} else if (irt_isnum(t)) {
setnumV(o, ex->fpr[r-RID_MIN_FPR]);
#endif
#if LJ_64
} else if (irt_islightud(t)) {
/* 64 bit lightuserdata which may escape already has the tag bits. */
o->u64 = ex->gpr[r-RID_MIN_GPR];
#endif
} else {
if (!irt_ispri(t))
setgcrefi(o->gcr, ex->gpr[r-RID_MIN_GPR]);
setitype(o, irt_toitype(t));
}
}
if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) {
rs = (ir+1)->prev;
if (LJ_UNLIKELY(bloomtest(rfilt, ref+1)))
rs = snap_renameref(T, snapno, ref+1, rs);
o->u32.hi = (ra_hasspill(regsp_spill(rs))) ?
(uint32_t)*&ex->spill[regsp_spill(rs)] :
(uint32_t)ex->gpr[regsp_reg(rs)-RID_MIN_GPR];
}
}
}
switch (bc_op(*pc)) {
case BC_CALLM: case BC_CALLMT: case BC_RETM: case BC_TSETM:
L->top = frame + nslots;
break;
default:
L->top = curr_topL(L);
break;
}
lua_assert(map + nent == flinks);
return pc;
}
/* Replay snapshot state to setup side trace. */
void lj_snap_replay(jit_State *J, GCtrace *T)
{
SnapShot *snap = &T->snap[J->exitno];
SnapEntry *map = &T->snapmap[snap->mapofs];
MSize n, nent = snap->nent;
BloomFilter seen = 0;
int pass23 = 0;
J->framedepth = 0;
/* Emit IR for slots inherited from parent snapshot. */
for (n = 0; n < nent; n++) {
SnapEntry sn = map[n];
BCReg s = snap_slot(sn);
IRRef ref = snap_ref(sn);
IRIns *ir = &T->ir[ref];
TRef tr;
/* The bloom filter avoids O(nent^2) overhead for de-duping slots. */
if (bloomtest(seen, ref) && (tr = snap_dedup(J, map, n, ref)) != 0)
goto setslot;
bloomset(seen, ref);
if (irref_isk(ref)) {
tr = snap_replay_const(J, ir);
} else if (!regsp_used(ir->prev)) {
pass23 = 1;
lua_assert(s != 0);
tr = s;
} else {
IRType t = irt_type(ir->t);
uint32_t mode = IRSLOAD_INHERIT|IRSLOAD_PARENT;
if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) t = IRT_NUM;
if (ir->o == IR_SLOAD) mode |= (ir->op2 & IRSLOAD_READONLY);
tr = emitir_raw(IRT(IR_SLOAD, t), s, mode);
}
setslot:
J->slot[s] = tr | (sn&(SNAP_CONT|SNAP_FRAME)); /* Same as TREF_* flags. */
J->framedepth += ((sn & (SNAP_CONT|SNAP_FRAME)) && s);
if ((sn & SNAP_FRAME))
J->baseslot = s+1;
}
if (pass23) {
IRIns *irlast = &T->ir[snap->ref];
pass23 = 0;
/* Emit dependent PVALs. */
for (n = 0; n < nent; n++) {
SnapEntry sn = map[n];
IRRef refp = snap_ref(sn);
IRIns *ir = &T->ir[refp];
if (regsp_reg(ir->r) == RID_SUNK) {
if (J->slot[snap_slot(sn)] != snap_slot(sn)) continue;
pass23 = 1;
lua_assert(ir->o == IR_TNEW || ir->o == IR_TDUP ||
ir->o == IR_CNEW || ir->o == IR_CNEWI);
if (ir->op1 >= T->nk) snap_pref(J, T, map, nent, seen, ir->op1);
if (ir->op2 >= T->nk) snap_pref(J, T, map, nent, seen, ir->op2);
if (LJ_HASFFI && ir->o == IR_CNEWI) {
if (LJ_32 && refp+1 < T->nins && (ir+1)->o == IR_HIOP)
snap_pref(J, T, map, nent, seen, (ir+1)->op2);
} else {
IRIns *irs;
for (irs = ir+1; irs < irlast; irs++)
if (irs->r == RID_SINK && snap_sunk_store(T, ir, irs)) {
if (snap_pref(J, T, map, nent, seen, irs->op2) == 0)
snap_pref(J, T, map, nent, seen, T->ir[irs->op2].op1);
else if ((LJ_SOFTFP || (LJ_32 && LJ_HASFFI)) &&
irs+1 < irlast && (irs+1)->o == IR_HIOP)
snap_pref(J, T, map, nent, seen, (irs+1)->op2);
}
}
} else if (!irref_isk(refp) && !regsp_used(ir->prev)) {
lua_assert(ir->o == IR_CONV && ir->op2 == IRCONV_NUM_INT);
J->slot[snap_slot(sn)] = snap_pref(J, T, map, nent, seen, ir->op1);
}
}
/* Replay sunk instructions. */
for (n = 0; pass23 && n < nent; n++) {
SnapEntry sn = map[n];
IRRef refp = snap_ref(sn);
IRIns *ir = &T->ir[refp];
if (regsp_reg(ir->r) == RID_SUNK) {
TRef op1, op2;
if (J->slot[snap_slot(sn)] != snap_slot(sn)) { /* De-dup allocs. */
J->slot[snap_slot(sn)] = J->slot[J->slot[snap_slot(sn)]];
continue;
}
op1 = ir->op1;
if (op1 >= T->nk) op1 = snap_pref(J, T, map, nent, seen, op1);
op2 = ir->op2;
if (op2 >= T->nk) op2 = snap_pref(J, T, map, nent, seen, op2);
if (LJ_HASFFI && ir->o == IR_CNEWI) {
if (LJ_32 && refp+1 < T->nins && (ir+1)->o == IR_HIOP) {
lj_needsplit(J); /* Emit joining HIOP. */
op2 = emitir_raw(IRT(IR_HIOP, IRT_I64), op2,
snap_pref(J, T, map, nent, seen, (ir+1)->op2));
}
J->slot[snap_slot(sn)] = emitir(ir->ot & ~(IRT_MARK|IRT_ISPHI), op1, op2);
} else {
IRIns *irs;
TRef tr = emitir(ir->ot, op1, op2);
J->slot[snap_slot(sn)] = tr;
for (irs = ir+1; irs < irlast; irs++)
if (irs->r == RID_SINK && snap_sunk_store(T, ir, irs)) {
IRIns *irr = &T->ir[irs->op1];
TRef val, key = irr->op2, tmp = tr;
if (irr->o != IR_FREF) {
IRIns *irk = &T->ir[key];
if (irr->o == IR_HREFK)
key = lj_ir_kslot(J, snap_replay_const(J, &T->ir[irk->op1]),
irk->op2);
else
key = snap_replay_const(J, irk);
if (irr->o == IR_HREFK || irr->o == IR_AREF) {
IRIns *irf = &T->ir[irr->op1];
tmp = emitir(irf->ot, tmp, irf->op2);
}
}
tmp = emitir(irr->ot, tmp, key);
val = snap_pref(J, T, map, nent, seen, irs->op2);
if (val == 0) {
IRIns *irc = &T->ir[irs->op2];
lua_assert(irc->o == IR_CONV && irc->op2 == IRCONV_NUM_INT);
val = snap_pref(J, T, map, nent, seen, irc->op1);
val = emitir(IRTN(IR_CONV), val, IRCONV_NUM_INT);
} else if ((LJ_SOFTFP || (LJ_32 && LJ_HASFFI)) &&
irs+1 < irlast && (irs+1)->o == IR_HIOP) {
IRType t = IRT_I64;
if (LJ_SOFTFP && irt_type((irs+1)->t) == IRT_SOFTFP)
t = IRT_NUM;
lj_needsplit(J);
if (irref_isk(irs->op2) && irref_isk((irs+1)->op2)) {
uint64_t k = (uint32_t)T->ir[irs->op2].i +
((uint64_t)T->ir[(irs+1)->op2].i << 32);
val = lj_ir_k64(J, t == IRT_I64 ? IR_KINT64 : IR_KNUM,
lj_ir_k64_find(J, k));
} else {
val = emitir_raw(IRT(IR_HIOP, t), val,
snap_pref(J, T, map, nent, seen, (irs+1)->op2));
}
tmp = emitir(IRT(irs->o, t), tmp, val);
continue;
}
tmp = emitir(irs->ot, tmp, val);
} else if (LJ_HASFFI && irs->o == IR_XBAR && ir->o == IR_CNEW) {
emitir(IRT(IR_XBAR, IRT_NIL), 0, 0);
}
}
}
}
}
J->base = J->slot + J->baseslot;
J->maxslot = snap->nslots - J->baseslot;
lj_snap_add(J);
if (pass23) /* Need explicit GC step _after_ initial snapshot. */
emitir_raw(IRTG(IR_GCSTEP, IRT_NIL), 0, 0);
}
/* Restore interpreter state from exit state with the help of a snapshot. */
void lj_snap_restore(jit_State *J, void *exptr)
{
ExitState *ex = (ExitState *)exptr;
SnapNo snapno = J->exitno; /* For now, snapno == exitno. */
Trace *T = J->trace[J->parent];
SnapShot *snap = &T->snap[snapno];
BCReg s, nslots = snap->nslots;
IRRef2 *map = &T->snapmap[snap->mapofs];
IRRef2 *flinks = map + nslots + snap->nframelinks;
TValue *o, *newbase, *ntop;
BloomFilter rfilt = snap_renamefilter(T, snapno);
lua_State *L = J->L;
/* Make sure the stack is big enough for the slots from the snapshot. */
if (L->base + nslots >= L->maxstack) {
L->top = curr_topL(L);
lj_state_growstack(L, nslots - curr_proto(L)->framesize);
}
/* Fill stack slots with data from the registers and spill slots. */
newbase = NULL;
ntop = L->base;
for (s = 0, o = L->base-1; s < nslots; s++, o++) {
IRRef ref = snap_ref(map[s]);
if (ref) {
IRIns *ir = &T->ir[ref];
if (irref_isk(ref)) { /* Restore constant slot. */
lj_ir_kvalue(L, o, ir);
} else {
IRType1 t = ir->t;
RegSP rs = ir->prev;
if (LJ_UNLIKELY(bloomtest(rfilt, ref)))
rs = snap_renameref(T, snapno, ref, rs);
if (ra_hasspill(regsp_spill(rs))) { /* Restore from spill slot. */
int32_t *sps = &ex->spill[regsp_spill(rs)];
if (irt_isinteger(t)) {
setintV(o, *sps);
} else if (irt_isnum(t)) {
o->u64 = *(uint64_t *)sps;
} else {
lua_assert(!irt_ispri(t)); /* PRI refs never have a spill slot. */
setgcrefi(o->gcr, *sps);
setitype(o, irt_toitype(t));
}
} else if (ra_hasreg(regsp_reg(rs))) { /* Restore from register. */
Reg r = regsp_reg(rs);
if (irt_isinteger(t)) {
setintV(o, ex->gpr[r-RID_MIN_GPR]);
} else if (irt_isnum(t)) {
setnumV(o, ex->fpr[r-RID_MIN_FPR]);
} else {
if (!irt_ispri(t))
setgcrefi(o->gcr, ex->gpr[r-RID_MIN_GPR]);
setitype(o, irt_toitype(t));
}
} else { /* Restore frame slot. */
lua_assert(ir->o == IR_FRAME);
/* This works for both PTR and FUNC IR_FRAME. */
setgcrefp(o->fr.func, mref(T->ir[ir->op2].ptr, void));
if (s != 0) /* Do not overwrite link to previous frame. */
o->fr.tp.ftsz = (int32_t)*--flinks;
if (irt_isfunc(ir->t)) {
GCfunc *fn = gco2func(gcref(T->ir[ir->op2].gcr));
if (isluafunc(fn)) {
TValue *fs;
newbase = o+1;
fs = newbase + funcproto(fn)->framesize;
if (fs > ntop) ntop = fs; /* Update top for newly added frames. */
}
}
}
}
} else if (newbase) {
setnilV(o); /* Clear unreferenced slots of newly added frames. */
}
}
