/* Clear array part of table. */
static LJ_AINLINE void clearapart(GCtab *t)
{
uint32_t i, asize = t->asize;
TValue *array = tvref(t->array);
for (i = 0; i < asize; i++)
setnilV(&array[i]);
}
/* Find error function for runtime errors. Requires an extra stack traversal. */
static ptrdiff_t finderrfunc(lua_State *L)
{
cTValue *frame = L->base-1, *bot = tvref(L->stack);
void *cf = L->cframe;
while (frame > bot && cf) {
while (cframe_nres(cframe_raw(cf)) < 0) { /* cframe without frame? */
if (frame >= restorestack(L, -cframe_nres(cf)))
break;
if (cframe_errfunc(cf) >= 0) /* Error handler not inherited (-1)? */
return cframe_errfunc(cf);
cf = cframe_prev(cf); /* Else unwind cframe and continue searching. */
if (cf == NULL)
return 0;
}
switch (frame_typep(frame)) {
case FRAME_LUA:
case FRAME_LUAP:
frame = frame_prevl(frame);
break;
case FRAME_C:
cf = cframe_prev(cf);
/* fallthrough */
case FRAME_VARG:
frame = frame_prevd(frame);
break;
case FRAME_CONT:
#if LJ_HASFFI
if ((frame-1)->u32.lo == LJ_CONT_FFI_CALLBACK)
cf = cframe_prev(cf);
#endif
frame = frame_prevd(frame);
break;
case FRAME_CP:
if (cframe_canyield(cf)) return 0;
if (cframe_errfunc(cf) >= 0)
return cframe_errfunc(cf);
frame = frame_prevd(frame);
break;
case FRAME_PCALL:
case FRAME_PCALLH:
if (frame_ftsz(frame) >= (ptrdiff_t)(2*sizeof(TValue))) /* xpcall? */
return savestack(L, frame-1); /* Point to xpcall's errorfunc. */
return 0;
default:
lua_assert(0);
return 0;
}
}
return 0;
}
/* Find error function for runtime errors. Requires an extra stack traversal. */
static ptrdiff_t finderrfunc(lua_State *L)
{
cTValue *frame = L->base-1, *bot = tvref(L->stack)+LJ_FR2;
void *cf = L->cframe;
while (frame > bot && cf) {
while (cframe_nres(cframe_raw(cf)) < 0) { /* cframe without frame? */
if (frame >= restorestack(L, -cframe_nres(cf)))
break;
if (cframe_errfunc(cf) >= 0) /* Error handler not inherited (-1)? */
return cframe_errfunc(cf);
cf = cframe_prev(cf); /* Else unwind cframe and continue searching. */
if (cf == NULL)
return 0;
}
switch (frame_typep(frame)) {
case FRAME_LUA:
case FRAME_LUAP:
frame = frame_prevl(frame);
break;
case FRAME_C:
cf = cframe_prev(cf);
/* fallthrough */
case FRAME_VARG:
frame = frame_prevd(frame);
break;
case FRAME_CONT:
if (frame_iscont_fficb(frame))
cf = cframe_prev(cf);
frame = frame_prevd(frame);
break;
case FRAME_CP:
if (cframe_canyield(cf)) return 0;
if (cframe_errfunc(cf) >= 0)
return cframe_errfunc(cf);
frame = frame_prevd(frame);
break;
case FRAME_PCALL:
case FRAME_PCALLH:
if (frame_func(frame_prevd(frame))->c.ffid == FF_xpcall)
return savestack(L, frame_prevd(frame)+1); /* xpcall's errorfunc. */
return 0;
default:
lua_assert(0);
return 0;
}
}
return 0;
}
/* Call a hook. */
static void callhook(lua_State *L, int event, BCLine line)
{
global_State *g = G(L);
lua_Hook hookf = g->hookf;
if (hookf && !hook_active(g)) {
lua_Debug ar;
lj_trace_abort(g); /* Abort recording on any hook call. */
ar.event = event;
ar.currentline = line;
/* Top frame, nextframe = NULL. */
ar.i_ci = cast_int((L->base-1) - tvref(L->stack));
lj_state_checkstack(L, 1+LUA_MINSTACK);
hook_enter(g);
hookf(L, &ar);
lua_assert(hook_active(g));
hook_leave(g);
}
}
/* Get frame corresponding to a level. */
cTValue *lj_debug_frame(lua_State *L, int level, int *size)
{
cTValue *frame, *nextframe, *bot = tvref(L->stack);
/* Traverse frames backwards. */
for (nextframe = frame = L->base-1; frame > bot; ) {
if (frame_gc(frame) == obj2gco(L))
level++; /* Skip dummy frames. See lj_meta_call(). */
if (level-- == 0) {
*size = (int)(nextframe - frame);
return frame; /* Level found. */
}
nextframe = frame;
if (frame_islua(frame)) {
frame = frame_prevl(frame);
} else {
if (frame_isvarg(frame))
level++; /* Skip vararg pseudo-frame. */
frame = frame_prevd(frame);
}
}
*size = level;
return NULL; /* Level not found. */
}
/* Unwind until stop frame. Optionally cleanup frames. */
static void *err_unwind(lua_State *L, void *stopcf, int errcode)
{
TValue *frame = L->base-1;
void *cf = L->cframe;
while (cf) {
int32_t nres = cframe_nres(cframe_raw(cf));
if (nres < 0) { /* C frame without Lua frame? */
TValue *top = restorestack(L, -nres);
if (frame < top) { /* Frame reached? */
if (errcode) {
L->cframe = cframe_prev(cf);
L->base = frame+1;
unwindstack(L, top);
}
return cf;
}
}
if (frame <= tvref(L->stack))
break;
switch (frame_typep(frame)) {
case FRAME_LUA: /* Lua frame. */
case FRAME_LUAP:
frame = frame_prevl(frame);
break;
case FRAME_C: /* C frame. */
#if LJ_UNWIND_EXT
if (errcode) {
L->cframe = cframe_prev(cf);
L->base = frame_prevd(frame) + 1;
unwindstack(L, frame);
} else if (cf != stopcf) {
cf = cframe_prev(cf);
frame = frame_prevd(frame);
break;
}
return NULL; /* Continue unwinding. */
#else
UNUSED(stopcf);
cf = cframe_prev(cf);
frame = frame_prevd(frame);
break;
#endif
case FRAME_CP: /* Protected C frame. */
if (cframe_canyield(cf)) { /* Resume? */
if (errcode) {
L->cframe = NULL;
L->status = (uint8_t)errcode;
}
return cf;
}
if (errcode) {
L->cframe = cframe_prev(cf);
L->base = frame_prevd(frame) + 1;
unwindstack(L, frame);
}
return cf;
case FRAME_CONT: /* Continuation frame. */
case FRAME_VARG: /* Vararg frame. */
frame = frame_prevd(frame);
break;
case FRAME_PCALL: /* FF pcall() frame. */
case FRAME_PCALLH: /* FF pcall() frame inside hook. */
if (errcode) {
if (errcode == LUA_YIELD) {
frame = frame_prevd(frame);
break;
}
if (frame_typep(frame) == FRAME_PCALL)
hook_leave(G(L));
L->cframe = cf;
L->base = frame_prevd(frame) + 1;
unwindstack(L, L->base);
}
return (void *)((intptr_t)cf | CFRAME_UNWIND_FF);
}
}
/* No C frame. */
if (errcode) {
L->cframe = NULL;
L->base = tvref(L->stack)+1;
unwindstack(L, L->base);
if (G(L)->panic)
G(L)->panic(L);
exit(EXIT_FAILURE);
}
return L; /* Anything non-NULL will do. */
}
/* Restore interpreter state from exit state with the help of a snapshot. */
const BCIns *lj_snap_restore(jit_State *J, void *exptr)
{
ExitState *ex = (ExitState *)exptr;
SnapNo snapno = J->exitno; /* For now, snapno == exitno. */
GCtrace *T = traceref(J, J->parent);
SnapShot *snap = &T->snap[snapno];
MSize n, nent = snap->nent;
SnapEntry *map = &T->snapmap[snap->mapofs];
SnapEntry *flinks = &T->snapmap[snap_nextofs(T, snap)-1];
ptrdiff_t ftsz0;
TValue *frame;
BloomFilter rfilt = snap_renamefilter(T, snapno);
const BCIns *pc = snap_pc(map[nent]);
lua_State *L = J->L;
/* Set interpreter PC to the next PC to get correct error messages. */
setcframe_pc(cframe_raw(L->cframe), pc+1);
/* Make sure the stack is big enough for the slots from the snapshot. */
if (LJ_UNLIKELY(L->base + snap->topslot >= tvref(L->maxstack))) {
L->top = curr_topL(L);
lj_state_growstack(L, snap->topslot - curr_proto(L)->framesize);
}
/* Fill stack slots with data from the registers and spill slots. */
frame = L->base-1;
ftsz0 = frame_ftsz(frame); /* Preserve link to previous frame in slot #0. */
for (n = 0; n < nent; n++) {
SnapEntry sn = map[n];
if (!(sn & SNAP_NORESTORE)) {
TValue *o = &frame[snap_slot(sn)];
IRRef ref = snap_ref(sn);
IRIns *ir = &T->ir[ref];
if (ir->r == RID_SUNK) {
MSize j;
for (j = 0; j < n; j++)
if (snap_ref(map[j]) == ref) { /* De-duplicate sunk allocations. */
copyTV(L, o, &frame[snap_slot(map[j])]);
goto dupslot;
}
snap_unsink(J, T, ex, snapno, rfilt, ir, o);
dupslot:
continue;
}
snap_restoreval(J, T, ex, snapno, rfilt, ref, o);
if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM) && tvisint(o)) {
TValue tmp;
snap_restoreval(J, T, ex, snapno, rfilt, ref+1, &tmp);
o->u32.hi = tmp.u32.lo;
} else if ((sn & (SNAP_CONT|SNAP_FRAME))) {
lua_assert(!LJ_FR2); /* TODO_FR2: store 64 bit PCs. */
/* Overwrite tag with frame link. */
setframe_ftsz(o, snap_slot(sn) != 0 ? (int32_t)*flinks-- : ftsz0);
L->base = o+1;
}
}
}
lua_assert(map + nent == flinks);
/* Compute current stack top. */
switch (bc_op(*pc)) {
default:
if (bc_op(*pc) < BC_FUNCF) {
L->top = curr_topL(L);
break;
}
/* fallthrough */
case BC_CALLM: case BC_CALLMT: case BC_RETM: case BC_TSETM:
L->top = frame + snap->nslots;
break;
}
return pc;
}
/* Restore interpreter state from exit state with the help of a snapshot. */
const BCIns *lj_snap_restore(jit_State *J, void *exptr)
{
ExitState *ex = (ExitState *)exptr;
SnapNo snapno = J->exitno; /* For now, snapno == exitno. */
GCtrace *T = traceref(J, J->parent);
SnapShot *snap = &T->snap[snapno];
MSize n, nent = snap->nent;
SnapEntry *map = &T->snapmap[snap->mapofs];
SnapEntry *flinks = map + nent + snap->depth;
int32_t ftsz0;
BCReg nslots = snap->nslots;
TValue *frame;
BloomFilter rfilt = snap_renamefilter(T, snapno);
const BCIns *pc = snap_pc(map[nent]);
lua_State *L = J->L;
/* Set interpreter PC to the next PC to get correct error messages. */
setcframe_pc(cframe_raw(L->cframe), pc+1);
/* Make sure the stack is big enough for the slots from the snapshot. */
if (LJ_UNLIKELY(L->base + nslots > tvref(L->maxstack))) {
L->top = curr_topL(L);
lj_state_growstack(L, nslots - curr_proto(L)->framesize);
}
/* Fill stack slots with data from the registers and spill slots. */
frame = L->base-1;
ftsz0 = frame_ftsz(frame); /* Preserve link to previous frame in slot #0. */
for (n = 0; n < nent; n++) {
SnapEntry sn = map[n];
IRRef ref = snap_ref(sn);
BCReg s = snap_slot(sn);
TValue *o = &frame[s]; /* Stack slots are relative to start frame. */
IRIns *ir = &T->ir[ref];
if (irref_isk(ref)) { /* Restore constant slot. */
lj_ir_kvalue(L, o, ir);
if ((sn & (SNAP_CONT|SNAP_FRAME))) {
/* Overwrite tag with frame link. */
o->fr.tp.ftsz = s != 0 ? (int32_t)*flinks-- : ftsz0;
if ((sn & SNAP_FRAME)) {
GCfunc *fn = ir_kfunc(ir);
if (isluafunc(fn)) {
MSize framesize = funcproto(fn)->framesize;
L->base = ++o;
if (LJ_UNLIKELY(o + framesize > tvref(L->maxstack))) {
ptrdiff_t fsave = savestack(L, frame);
L->top = o;
lj_state_growstack(L, framesize); /* Grow again. */
frame = restorestack(L, fsave);
}
}
}
}
} else if (!(sn & SNAP_NORESTORE)) {
IRType1 t = ir->t;
RegSP rs = ir->prev;
lua_assert(!(sn & (SNAP_CONT|SNAP_FRAME)));
if (LJ_UNLIKELY(bloomtest(rfilt, ref)))
rs = snap_renameref(T, snapno, ref, rs);
if (ra_hasspill(regsp_spill(rs))) { /* Restore from spill slot. */
int32_t *sps = &ex->spill[regsp_spill(rs)];
if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) {
o->u32.lo = (uint32_t)*sps;
} else if (irt_isinteger(t)) {
setintV(o, *sps);
#if !LJ_SOFTFP
} else if (irt_isnum(t)) {
o->u64 = *(uint64_t *)sps;
#endif
#if LJ_64
} else if (irt_islightud(t)) {
/* 64 bit lightuserdata which may escape already has the tag bits. */
o->u64 = *(uint64_t *)sps;
#endif
} else {
lua_assert(!irt_ispri(t)); /* PRI refs never have a spill slot. */
setgcrefi(o->gcr, *sps);
setitype(o, irt_toitype(t));
}
} else { /* Restore from register. */
Reg r = regsp_reg(rs);
lua_assert(ra_hasreg(r));
if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) {
o->u32.lo = (uint32_t)ex->gpr[r-RID_MIN_GPR];
} else if (irt_isinteger(t)) {
setintV(o, (int32_t)ex->gpr[r-RID_MIN_GPR]);
#if !LJ_SOFTFP
} else if (irt_isnum(t)) {
setnumV(o, ex->fpr[r-RID_MIN_FPR]);
#endif
#if LJ_64
} else if (irt_islightud(t)) {
/* 64 bit lightuserdata which may escape already has the tag bits. */
o->u64 = ex->gpr[r-RID_MIN_GPR];
#endif
} else {
if (!irt_ispri(t))
setgcrefi(o->gcr, ex->gpr[r-RID_MIN_GPR]);
setitype(o, irt_toitype(t));
}
}
if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) {
rs = (ir+1)->prev;
if (LJ_UNLIKELY(bloomtest(rfilt, ref+1)))
rs = snap_renameref(T, snapno, ref+1, rs);
o->u32.hi = (ra_hasspill(regsp_spill(rs))) ?
(uint32_t)*&ex->spill[regsp_spill(rs)] :
(uint32_t)ex->gpr[regsp_reg(rs)-RID_MIN_GPR];
}
}
}
switch (bc_op(*pc)) {
case BC_CALLM: case BC_CALLMT: case BC_RETM: case BC_TSETM:
L->top = frame + nslots;
break;
default:
L->top = curr_topL(L);
break;
}
lua_assert(map + nent == flinks);
return pc;
}