/* Copy-substitute snapshot. */
static void loop_subst_snap(jit_State *J, SnapShot *osnap,
SnapEntry *loopmap, IRRef1 *subst)
{
SnapEntry *nmap, *omap = &J->cur.snapmap[osnap->mapofs];
SnapEntry *nextmap = &J->cur.snapmap[snap_nextofs(&J->cur, osnap)];
MSize nmapofs;
MSize on, ln, nn, onent = osnap->nent;
BCReg nslots = osnap->nslots;
SnapShot *snap = &J->cur.snap[J->cur.nsnap];
if (irt_isguard(J->guardemit)) { /* Guard inbetween? */
nmapofs = J->cur.nsnapmap;
J->cur.nsnap++; /* Add new snapshot. */
} else { /* Otherwise overwrite previous snapshot. */
snap--;
nmapofs = snap->mapofs;
}
J->guardemit.irt = 0;
/* Setup new snapshot. */
snap->mapofs = (uint16_t)nmapofs;
snap->ref = (IRRef1)J->cur.nins;
snap->nslots = nslots;
snap->topslot = osnap->topslot;
snap->count = 0;
nmap = &J->cur.snapmap[nmapofs];
/* Substitute snapshot slots. */
on = ln = nn = 0;
while (on < onent) {
SnapEntry osn = omap[on], lsn = loopmap[ln];
if (snap_slot(lsn) < snap_slot(osn)) { /* Copy slot from loop map. */
nmap[nn++] = lsn;
ln++;
} else { /* Copy substituted slot from snapshot map. */
if (snap_slot(lsn) == snap_slot(osn)) ln++; /* Shadowed loop slot. */
if (!irref_isk(snap_ref(osn)))
osn = snap_setref(osn, subst[snap_ref(osn)]);
nmap[nn++] = osn;
on++;
}
}
while (snap_slot(loopmap[ln]) < nslots) /* Copy remaining loop slots. */
nmap[nn++] = loopmap[ln++];
snap->nent = (uint8_t)nn;
omap += onent;
nmap += nn;
while (omap < nextmap) /* Copy PC + frame links. */
*nmap++ = *omap++;
J->cur.nsnapmap = (uint16_t)(nmap - J->cur.snapmap);
}
/* Copy RegSP from parent snapshot to the parent links of the IR. */
IRIns *lj_snap_regspmap(GCtrace *T, SnapNo snapno, IRIns *ir)
{
SnapShot *snap = &T->snap[snapno];
SnapEntry *map = &T->snapmap[snap->mapofs];
BloomFilter rfilt = snap_renamefilter(T, snapno);
MSize n = 0;
IRRef ref = 0;
for ( ; ; ir++) {
uint32_t rs;
if (ir->o == IR_SLOAD) {
if (!(ir->op2 & IRSLOAD_PARENT)) break;
for ( ; ; n++) {
lua_assert(n < snap->nent);
if (snap_slot(map[n]) == ir->op1) {
ref = snap_ref(map[n++]);
break;
}
}
} else if (LJ_SOFTFP && ir->o == IR_HIOP) {
ref++;
} else if (ir->o == IR_PVAL) {
ref = ir->op1 + REF_BIAS;
} else {
break;
}
rs = T->ir[ref].prev;
if (bloomtest(rfilt, ref))
rs = snap_renameref(T, snapno, ref, rs);
ir->prev = (uint16_t)rs;
lua_assert(regsp_used(rs));
}
return ir;
}
/* De-duplicate parent reference. */
static TRef snap_dedup(jit_State *J, SnapEntry *map, MSize nmax, IRRef ref)
{
MSize j;
for (j = 0; j < nmax; j++)
if (snap_ref(map[j]) == ref)
return J->slot[snap_slot(map[j])] & ~(SNAP_CONT|SNAP_FRAME);
return 0;
}
/* Convert a snapshot into a linear slot -> RegSP map.
** Note: unused slots are not initialized!
*/
void lj_snap_regspmap(uint16_t *rsmap, GCtrace *T, SnapNo snapno)
{
SnapShot *snap = &T->snap[snapno];
MSize n, nent = snap->nent;
SnapEntry *map = &T->snapmap[snap->mapofs];
BloomFilter rfilt = snap_renamefilter(T, snapno);
for (n = 0; n < nent; n++) {
SnapEntry sn = map[n];
IRRef ref = snap_ref(sn);
if (!irref_isk(ref)) {
IRIns *ir = &T->ir[ref];
uint32_t rs = ir->prev;
if (bloomtest(rfilt, ref))
rs = snap_renameref(T, snapno, ref, rs);
rsmap[snap_slot(sn)] = (uint16_t)rs;
}
}
}
/* Shrink last snapshot. */
void lj_snap_shrink(jit_State *J)
{
SnapShot *snap = &J->cur.snap[J->cur.nsnap-1];
SnapEntry *map = &J->cur.snapmap[snap->mapofs];
MSize n, m, nlim, nent = snap->nent;
uint8_t udf[SNAP_USEDEF_SLOTS];
BCReg maxslot = J->maxslot;
BCReg minslot = snap_usedef(J, udf, snap_pc(map[nent]), maxslot);
BCReg baseslot = J->baseslot;
maxslot += baseslot;
minslot += baseslot;
snap->nslots = (uint8_t)maxslot;
for (n = m = 0; n < nent; n++) { /* Remove unused slots from snapshot. */
BCReg s = snap_slot(map[n]);
if (s < minslot || (s < maxslot && udf[s-baseslot] == 0))
map[m++] = map[n]; /* Only copy used slots. */
}
snap->nent = (uint8_t)m;
nlim = J->cur.nsnapmap - snap->mapofs - 1;
while (n <= nlim) map[m++] = map[n++]; /* Move PC + frame links down. */
J->cur.nsnapmap = (uint16_t)(snap->mapofs + m); /* Free up space in map. */
}
/* Restore interpreter state from exit state with the help of a snapshot. */
const BCIns *lj_snap_restore(jit_State *J, void *exptr)
{
ExitState *ex = (ExitState *)exptr;
SnapNo snapno = J->exitno; /* For now, snapno == exitno. */
GCtrace *T = traceref(J, J->parent);
SnapShot *snap = &T->snap[snapno];
MSize n, nent = snap->nent;
SnapEntry *map = &T->snapmap[snap->mapofs];
SnapEntry *flinks = &T->snapmap[snap_nextofs(T, snap)-1];
ptrdiff_t ftsz0;
TValue *frame;
BloomFilter rfilt = snap_renamefilter(T, snapno);
const BCIns *pc = snap_pc(map[nent]);
lua_State *L = J->L;
/* Set interpreter PC to the next PC to get correct error messages. */
setcframe_pc(cframe_raw(L->cframe), pc+1);
/* Make sure the stack is big enough for the slots from the snapshot. */
if (LJ_UNLIKELY(L->base + snap->topslot >= tvref(L->maxstack))) {
L->top = curr_topL(L);
lj_state_growstack(L, snap->topslot - curr_proto(L)->framesize);
}
/* Fill stack slots with data from the registers and spill slots. */
frame = L->base-1;
ftsz0 = frame_ftsz(frame); /* Preserve link to previous frame in slot #0. */
for (n = 0; n < nent; n++) {
SnapEntry sn = map[n];
if (!(sn & SNAP_NORESTORE)) {
TValue *o = &frame[snap_slot(sn)];
IRRef ref = snap_ref(sn);
IRIns *ir = &T->ir[ref];
if (ir->r == RID_SUNK) {
MSize j;
for (j = 0; j < n; j++)
if (snap_ref(map[j]) == ref) { /* De-duplicate sunk allocations. */
copyTV(L, o, &frame[snap_slot(map[j])]);
goto dupslot;
}
snap_unsink(J, T, ex, snapno, rfilt, ir, o);
dupslot:
continue;
}
snap_restoreval(J, T, ex, snapno, rfilt, ref, o);
if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM) && tvisint(o)) {
TValue tmp;
snap_restoreval(J, T, ex, snapno, rfilt, ref+1, &tmp);
o->u32.hi = tmp.u32.lo;
} else if ((sn & (SNAP_CONT|SNAP_FRAME))) {
lua_assert(!LJ_FR2); /* TODO_FR2: store 64 bit PCs. */
/* Overwrite tag with frame link. */
setframe_ftsz(o, snap_slot(sn) != 0 ? (int32_t)*flinks-- : ftsz0);
L->base = o+1;
}
}
}
lua_assert(map + nent == flinks);
/* Compute current stack top. */
switch (bc_op(*pc)) {
default:
if (bc_op(*pc) < BC_FUNCF) {
L->top = curr_topL(L);
break;
}
/* fallthrough */
case BC_CALLM: case BC_CALLMT: case BC_RETM: case BC_TSETM:
L->top = frame + snap->nslots;
break;
}
return pc;
}
/* Replay snapshot state to setup side trace. */
void lj_snap_replay(jit_State *J, GCtrace *T)
{
SnapShot *snap = &T->snap[J->exitno];
SnapEntry *map = &T->snapmap[snap->mapofs];
MSize n, nent = snap->nent;
BloomFilter seen = 0;
int pass23 = 0;
J->framedepth = 0;
/* Emit IR for slots inherited from parent snapshot. */
for (n = 0; n < nent; n++) {
SnapEntry sn = map[n];
BCReg s = snap_slot(sn);
IRRef ref = snap_ref(sn);
IRIns *ir = &T->ir[ref];
TRef tr;
/* The bloom filter avoids O(nent^2) overhead for de-duping slots. */
if (bloomtest(seen, ref) && (tr = snap_dedup(J, map, n, ref)) != 0)
goto setslot;
bloomset(seen, ref);
if (irref_isk(ref)) {
tr = snap_replay_const(J, ir);
} else if (!regsp_used(ir->prev)) {
pass23 = 1;
lua_assert(s != 0);
tr = s;
} else {
IRType t = irt_type(ir->t);
uint32_t mode = IRSLOAD_INHERIT|IRSLOAD_PARENT;
if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) t = IRT_NUM;
if (ir->o == IR_SLOAD) mode |= (ir->op2 & IRSLOAD_READONLY);
tr = emitir_raw(IRT(IR_SLOAD, t), s, mode);
}
setslot:
J->slot[s] = tr | (sn&(SNAP_CONT|SNAP_FRAME)); /* Same as TREF_* flags. */
J->framedepth += ((sn & (SNAP_CONT|SNAP_FRAME)) && s);
if ((sn & SNAP_FRAME))
J->baseslot = s+1;
}
if (pass23) {
IRIns *irlast = &T->ir[snap->ref];
pass23 = 0;
/* Emit dependent PVALs. */
for (n = 0; n < nent; n++) {
SnapEntry sn = map[n];
IRRef refp = snap_ref(sn);
IRIns *ir = &T->ir[refp];
if (regsp_reg(ir->r) == RID_SUNK) {
if (J->slot[snap_slot(sn)] != snap_slot(sn)) continue;
pass23 = 1;
lua_assert(ir->o == IR_TNEW || ir->o == IR_TDUP ||
ir->o == IR_CNEW || ir->o == IR_CNEWI);
if (ir->op1 >= T->nk) snap_pref(J, T, map, nent, seen, ir->op1);
if (ir->op2 >= T->nk) snap_pref(J, T, map, nent, seen, ir->op2);
if (LJ_HASFFI && ir->o == IR_CNEWI) {
if (LJ_32 && refp+1 < T->nins && (ir+1)->o == IR_HIOP)
snap_pref(J, T, map, nent, seen, (ir+1)->op2);
} else {
IRIns *irs;
for (irs = ir+1; irs < irlast; irs++)
if (irs->r == RID_SINK && snap_sunk_store(T, ir, irs)) {
if (snap_pref(J, T, map, nent, seen, irs->op2) == 0)
snap_pref(J, T, map, nent, seen, T->ir[irs->op2].op1);
else if ((LJ_SOFTFP || (LJ_32 && LJ_HASFFI)) &&
irs+1 < irlast && (irs+1)->o == IR_HIOP)
snap_pref(J, T, map, nent, seen, (irs+1)->op2);
}
}
} else if (!irref_isk(refp) && !regsp_used(ir->prev)) {
lua_assert(ir->o == IR_CONV && ir->op2 == IRCONV_NUM_INT);
J->slot[snap_slot(sn)] = snap_pref(J, T, map, nent, seen, ir->op1);
}
}
/* Replay sunk instructions. */
for (n = 0; pass23 && n < nent; n++) {
SnapEntry sn = map[n];
IRRef refp = snap_ref(sn);
IRIns *ir = &T->ir[refp];
if (regsp_reg(ir->r) == RID_SUNK) {
TRef op1, op2;
if (J->slot[snap_slot(sn)] != snap_slot(sn)) { /* De-dup allocs. */
J->slot[snap_slot(sn)] = J->slot[J->slot[snap_slot(sn)]];
continue;
}
op1 = ir->op1;
if (op1 >= T->nk) op1 = snap_pref(J, T, map, nent, seen, op1);
op2 = ir->op2;
if (op2 >= T->nk) op2 = snap_pref(J, T, map, nent, seen, op2);
if (LJ_HASFFI && ir->o == IR_CNEWI) {
if (LJ_32 && refp+1 < T->nins && (ir+1)->o == IR_HIOP) {
lj_needsplit(J); /* Emit joining HIOP. */
op2 = emitir_raw(IRT(IR_HIOP, IRT_I64), op2,
snap_pref(J, T, map, nent, seen, (ir+1)->op2));
}
J->slot[snap_slot(sn)] = emitir(ir->ot & ~(IRT_MARK|IRT_ISPHI), op1, op2);
} else {
IRIns *irs;
TRef tr = emitir(ir->ot, op1, op2);
J->slot[snap_slot(sn)] = tr;
for (irs = ir+1; irs < irlast; irs++)
if (irs->r == RID_SINK && snap_sunk_store(T, ir, irs)) {
IRIns *irr = &T->ir[irs->op1];
TRef val, key = irr->op2, tmp = tr;
if (irr->o != IR_FREF) {
IRIns *irk = &T->ir[key];
if (irr->o == IR_HREFK)
key = lj_ir_kslot(J, snap_replay_const(J, &T->ir[irk->op1]),
irk->op2);
else
key = snap_replay_const(J, irk);
if (irr->o == IR_HREFK || irr->o == IR_AREF) {
IRIns *irf = &T->ir[irr->op1];
tmp = emitir(irf->ot, tmp, irf->op2);
}
}
tmp = emitir(irr->ot, tmp, key);
val = snap_pref(J, T, map, nent, seen, irs->op2);
if (val == 0) {
IRIns *irc = &T->ir[irs->op2];
lua_assert(irc->o == IR_CONV && irc->op2 == IRCONV_NUM_INT);
val = snap_pref(J, T, map, nent, seen, irc->op1);
val = emitir(IRTN(IR_CONV), val, IRCONV_NUM_INT);
} else if ((LJ_SOFTFP || (LJ_32 && LJ_HASFFI)) &&
irs+1 < irlast && (irs+1)->o == IR_HIOP) {
IRType t = IRT_I64;
if (LJ_SOFTFP && irt_type((irs+1)->t) == IRT_SOFTFP)
t = IRT_NUM;
lj_needsplit(J);
if (irref_isk(irs->op2) && irref_isk((irs+1)->op2)) {
uint64_t k = (uint32_t)T->ir[irs->op2].i +
((uint64_t)T->ir[(irs+1)->op2].i << 32);
val = lj_ir_k64(J, t == IRT_I64 ? IR_KINT64 : IR_KNUM,
lj_ir_k64_find(J, k));
} else {
val = emitir_raw(IRT(IR_HIOP, t), val,
snap_pref(J, T, map, nent, seen, (irs+1)->op2));
}
tmp = emitir(IRT(irs->o, t), tmp, val);
continue;
}
tmp = emitir(irs->ot, tmp, val);
} else if (LJ_HASFFI && irs->o == IR_XBAR && ir->o == IR_CNEW) {
emitir(IRT(IR_XBAR, IRT_NIL), 0, 0);
}
}
}
}
}
J->base = J->slot + J->baseslot;
J->maxslot = snap->nslots - J->baseslot;
lj_snap_add(J);
if (pass23) /* Need explicit GC step _after_ initial snapshot. */
emitir_raw(IRTG(IR_GCSTEP, IRT_NIL), 0, 0);
}
int
irEngine(Capability *cap, Fragment *F)
{
static Inst disp[] = {
#define IRIMPL(name, f, o1, o2) &&op_##name,
IRDEF(IRIMPL)
#undef IRIMPL
&&stop
};
IRRef ref;
Thread *T = cap->T;
Word nphis = F->nphis;
Word *base = T->base - 1;
Word szins = F->nins - F->nk;
Word vals_[szins + nphis];
Word *phibuf = &vals_[szins]; /* For parallel copy of PHI nodes */
Word *vals = vals_ - (int)F->nk;
IRIns *pc = F->ir + REF_FIRST;
IRRef pcref = REF_FIRST;
IRIns *pcmax = F->ir + F->nins;
IRIns *pcloop = F->nloop ? F->ir + F->nloop + 1 : pc;
//int count = 100;
DBG_PR("*** Executing trace.\n"
"*** base = %p\n"
"*** pc = %p\n"
"*** pcmax = %p (%d)\n"
"*** loop = %p (%d)\n",
base, pc, pcmax, (int)(pcmax - pc), pcloop, (int)(pcloop - pc));
for (ref = F->nk; ref < REF_BIAS; ref++) {
switch (IR(ref)->o) {
case IR_KINT: vals[ref] = (Word)IR(ref)->i; break;
case IR_KBASEO: vals[ref] = (Word)(T->base + IR(ref)->i); break;
case IR_KWORD: vals[ref] = (Word)(F->kwords[IR(ref)->u]); break;
default:
LC_ASSERT(0); break;
}
DBG_LVL(2, "%d, %" FMT_WordX "\n", ref - REF_BIAS, vals[ref]);
}
vals[REF_BASE] = (Word)base;
goto *disp[pc->o];
# define DISPATCH_NEXT \
if (irt_type(pc->t) != IRT_VOID && pc->o != IR_PHI) { \
if (irt_type(pc->t) == IRT_I32) \
DBG_LVL(2, " ===> %" FMT_Int "\n", vals[pcref]); \
else \
DBG_LVL(2, " ===> 0x%" FMT_WordX "\n", vals[pcref]); } \
++pc; ++pcref; \
if (LC_UNLIKELY(pc >= pcmax)) { pc = pcloop; pcref = F->nloop + 1; } \
if (pc->o != IR_NOP) { \
DBG_LVL(2, "[%d] ", pcref - REF_BIAS); \
IF_DBG_LVL(2, printIR(F, *pc)); } \
goto *disp[pc->o]
op_NOP:
op_FRAME:
op_RET:
op_LOOP:
DISPATCH_NEXT;
op_PHI:
{
/* PHI nodes represent parallel assignments, so as soon as we
discover the first PHI node, we perform all assignments in
parallel. */
LC_ASSERT(nphis > 0);
u2 i;
DBG_LVL(3, " ( ");
for (i = 0; i < nphis; i++) {
DBG_LVL(3, "%d ", irref_int(pc[i].op2));
phibuf[i] = vals[pc[i].op2];
}
DBG_LVL(3, ") --> ( ");
for (i = 0; i < nphis; i++) {
DBG_LVL(3, "%d ", irref_int(pc[i].op1));
vals[pc[i].op1] = phibuf[i];
}
DBG_LVL(3, ") [%d phis]\n", (int)nphis);
pc += nphis - 1;
//vals[pc->op1] = vals[pc->op2];
DISPATCH_NEXT;
}
op_LT:
recordEvent(EV_CMP, 0);
if (!((WordInt)vals[pc->op1] < (WordInt)vals[pc->op2]))
goto guard_failed;
DISPATCH_NEXT;
op_GE:
recordEvent(EV_CMP, 0);
if (!((WordInt)vals[pc->op1] >= (WordInt)vals[pc->op2]))
goto guard_failed;
DISPATCH_NEXT;
op_LE:
recordEvent(EV_CMP, 0);
if (!((WordInt)vals[pc->op1] <= (WordInt)vals[pc->op2]))
goto guard_failed;
DISPATCH_NEXT;
op_GT:
recordEvent(EV_CMP, 0);
if (!((WordInt)vals[pc->op1] > (WordInt)vals[pc->op2]))
goto guard_failed;
DISPATCH_NEXT;
op_EQ:
recordEvent(EV_CMP, 0);
if (!((WordInt)vals[pc->op1] == (WordInt)vals[pc->op2])) {
goto guard_failed;
}
DISPATCH_NEXT;
op_NE:
recordEvent(EV_CMP, 0);
if (!((WordInt)vals[pc->op1] != (WordInt)vals[pc->op2]))
goto guard_failed;
DISPATCH_NEXT;
op_ADD:
recordEvent(EV_ALU, 0);
vals[pcref] = vals[pc->op1] + vals[pc->op2];
DISPATCH_NEXT;
op_SUB:
recordEvent(EV_ALU, 0);
vals[pcref] = vals[pc->op1] - vals[pc->op2];
DISPATCH_NEXT;
op_MUL:
recordEvent(EV_MUL, 0);
vals[pcref] = (WordInt)vals[pc->op1] * (WordInt)vals[pc->op2];
DISPATCH_NEXT;
op_DIV:
recordEvent(EV_REMDIV, 0);
if (LC_LIKELY(vals[pc->op2] != 0))
vals[pcref] = (WordInt)vals[pc->op1] / (WordInt)vals[pc->op2];
else
LC_ASSERT(0);
DISPATCH_NEXT;
op_REM:
recordEvent(EV_REMDIV, 0);
if (LC_LIKELY(vals[pc->op2] != 0))
vals[pcref] = (WordInt)vals[pc->op1] % (WordInt)vals[pc->op2];
else
LC_ASSERT(0);
DISPATCH_NEXT;
op_FREF:
vals[pcref] = (Word)(((Closure*)vals[pc->op1])->payload + (pc->op2 - 1));
DISPATCH_NEXT;
op_FLOAD:
recordEvent(EV_LOAD, 0);
vals[pcref] = *((Word*)vals[pc->op1]);
DISPATCH_NEXT;
op_SLOAD:
recordEvent(EV_LOAD, 0);
vals[pcref] = base[pc->op1];
DISPATCH_NEXT;
op_ILOAD:
recordEvent(EV_LOAD, 0);
vals[pcref] = (Word)getInfo(vals[pc->op1]);
DISPATCH_NEXT;
op_NEW:
if (!ir_issunken(pc)) {
// do actual allocation on trace
HeapInfo *hp = &F->heap[pc->op2];
int j;
recordEvent(EV_ALLOC, hp->nfields + 1);
Closure *cl = allocClosure(wordsof(ClosureHeader) + hp->nfields);
setInfo(cl, (InfoTable*)vals[pc->op1]);
for (j = 0; j < hp->nfields; j++) {
cl->payload[j] = vals[getHeapInfoField(F, hp, j)];
}
vals[pcref] = (Word)cl;
} else {
vals[pcref] = 0; // to trigger an error if accessed
}
DISPATCH_NEXT;
op_UPDATE:
{
recordEvent(EV_UPDATE, 0);
Closure *oldnode = (Closure *)vals[pc->op1];
Closure *newnode = (Closure *)base[pc->op2];
setInfo(oldnode, (InfoTable*)&stg_IND_info);
oldnode->payload[0] = (Word)newnode;
DISPATCH_NEXT;
}
op_RLOAD:
op_FSTORE:
op_RENAME:
op_BNOT: op_BAND: op_BOR: op_BXOR:
op_BSHL: op_BSHR: op_BSAR:
op_BROL: op_BROR:
// These should never be executed.
op_BASE:
op_KINT:
op_KWORD:
op_KBASEO:
LC_ASSERT(0);
guard_failed:
DBG_PR("Exiting at %d\n", pcref - REF_BIAS);
{
int i;
SnapShot *snap = 0;
SnapEntry *se;
for (i = 0; i < F->nsnap; i++) {
if (F->snap[i].ref == pcref) {
snap = &F->snap[i];
break;
}
}
LC_ASSERT(snap != 0);
snap->count++;
se = F->snapmap + snap->mapofs;
DBG_PR("Snapshot: %d, Snap entries: %d, slots = %d\n",
i, snap->nent, snap->nslots);
recordEvent(EV_EXIT, snap->nent);
for (i = 0; i < snap->nent; i++, se++) {
BCReg s = snap_slot(*se);
IRRef r = snap_ref(*se);
DBG_PR("base[%d] = ", s - 1);
base[s] = restoreValue(F, vals, r);
IF_DBG_LVL(1, printSlot(stderr, base + s); fprintf(stderr, "\n"));
//DBG_PR("0x%" FMT_WordX "\n", base[s]);
}
DBG_PR("Base slot: %d\n", se[1]);
// se[1] =
T->pc = (BCIns *)F->startpc + (int)se[0];
T->base = base + se[1];
T->top = base + snap->nslots;
//printFrame(T->base, T->top);
return 0;
}
stop:
return 1;
}
/* Restore interpreter state from exit state with the help of a snapshot. */
const BCIns *lj_snap_restore(jit_State *J, void *exptr)
{
ExitState *ex = (ExitState *)exptr;
SnapNo snapno = J->exitno; /* For now, snapno == exitno. */
GCtrace *T = traceref(J, J->parent);
SnapShot *snap = &T->snap[snapno];
MSize n, nent = snap->nent;
SnapEntry *map = &T->snapmap[snap->mapofs];
SnapEntry *flinks = map + nent + snap->depth;
int32_t ftsz0;
BCReg nslots = snap->nslots;
TValue *frame;
BloomFilter rfilt = snap_renamefilter(T, snapno);
const BCIns *pc = snap_pc(map[nent]);
lua_State *L = J->L;
/* Set interpreter PC to the next PC to get correct error messages. */
setcframe_pc(cframe_raw(L->cframe), pc+1);
/* Make sure the stack is big enough for the slots from the snapshot. */
if (LJ_UNLIKELY(L->base + nslots > tvref(L->maxstack))) {
L->top = curr_topL(L);
lj_state_growstack(L, nslots - curr_proto(L)->framesize);
}
/* Fill stack slots with data from the registers and spill slots. */
frame = L->base-1;
ftsz0 = frame_ftsz(frame); /* Preserve link to previous frame in slot #0. */
for (n = 0; n < nent; n++) {
SnapEntry sn = map[n];
IRRef ref = snap_ref(sn);
BCReg s = snap_slot(sn);
TValue *o = &frame[s]; /* Stack slots are relative to start frame. */
IRIns *ir = &T->ir[ref];
if (irref_isk(ref)) { /* Restore constant slot. */
lj_ir_kvalue(L, o, ir);
if ((sn & (SNAP_CONT|SNAP_FRAME))) {
/* Overwrite tag with frame link. */
o->fr.tp.ftsz = s != 0 ? (int32_t)*flinks-- : ftsz0;
if ((sn & SNAP_FRAME)) {
GCfunc *fn = ir_kfunc(ir);
if (isluafunc(fn)) {
MSize framesize = funcproto(fn)->framesize;
L->base = ++o;
if (LJ_UNLIKELY(o + framesize > tvref(L->maxstack))) {
ptrdiff_t fsave = savestack(L, frame);
L->top = o;
lj_state_growstack(L, framesize); /* Grow again. */
frame = restorestack(L, fsave);
}
}
}
}
} else if (!(sn & SNAP_NORESTORE)) {
IRType1 t = ir->t;
RegSP rs = ir->prev;
lua_assert(!(sn & (SNAP_CONT|SNAP_FRAME)));
if (LJ_UNLIKELY(bloomtest(rfilt, ref)))
rs = snap_renameref(T, snapno, ref, rs);
if (ra_hasspill(regsp_spill(rs))) { /* Restore from spill slot. */
int32_t *sps = &ex->spill[regsp_spill(rs)];
if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) {
o->u32.lo = (uint32_t)*sps;
} else if (irt_isinteger(t)) {
setintV(o, *sps);
#if !LJ_SOFTFP
} else if (irt_isnum(t)) {
o->u64 = *(uint64_t *)sps;
#endif
#if LJ_64
} else if (irt_islightud(t)) {
/* 64 bit lightuserdata which may escape already has the tag bits. */
o->u64 = *(uint64_t *)sps;
#endif
} else {
lua_assert(!irt_ispri(t)); /* PRI refs never have a spill slot. */
setgcrefi(o->gcr, *sps);
setitype(o, irt_toitype(t));
}
} else { /* Restore from register. */
Reg r = regsp_reg(rs);
lua_assert(ra_hasreg(r));
if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) {
o->u32.lo = (uint32_t)ex->gpr[r-RID_MIN_GPR];
} else if (irt_isinteger(t)) {
setintV(o, (int32_t)ex->gpr[r-RID_MIN_GPR]);
#if !LJ_SOFTFP
} else if (irt_isnum(t)) {
setnumV(o, ex->fpr[r-RID_MIN_FPR]);
#endif
#if LJ_64
} else if (irt_islightud(t)) {
/* 64 bit lightuserdata which may escape already has the tag bits. */
o->u64 = ex->gpr[r-RID_MIN_GPR];
#endif
} else {
if (!irt_ispri(t))
setgcrefi(o->gcr, ex->gpr[r-RID_MIN_GPR]);
setitype(o, irt_toitype(t));
}
}
if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) {
rs = (ir+1)->prev;
if (LJ_UNLIKELY(bloomtest(rfilt, ref+1)))
rs = snap_renameref(T, snapno, ref+1, rs);
o->u32.hi = (ra_hasspill(regsp_spill(rs))) ?
(uint32_t)*&ex->spill[regsp_spill(rs)] :
(uint32_t)ex->gpr[regsp_reg(rs)-RID_MIN_GPR];
}
}
}
switch (bc_op(*pc)) {
case BC_CALLM: case BC_CALLMT: case BC_RETM: case BC_TSETM:
L->top = frame + nslots;
break;
default:
L->top = curr_topL(L);
break;
}
lua_assert(map + nent == flinks);
return pc;
}
