/* Add frame links at the end of the snapshot. */
static BCReg snapshot_framelinks(jit_State *J, SnapEntry *map)
{
cTValue *frame = J->L->base - 1;
cTValue *lim = J->L->base - J->baseslot;
GCfunc *fn = frame_func(frame);
cTValue *ftop = isluafunc(fn) ? (frame+funcproto(fn)->framesize) : J->L->top;
MSize f = 0;
lua_assert(!LJ_FR2); /* TODO_FR2: store 64 bit PCs. */
map[f++] = SNAP_MKPC(J->pc); /* The current PC is always the first entry. */
while (frame > lim) { /* Backwards traversal of all frames above base. */
if (frame_islua(frame)) {
map[f++] = SNAP_MKPC(frame_pc(frame));
frame = frame_prevl(frame);
} else if (frame_iscont(frame)) {
map[f++] = SNAP_MKFTSZ(frame_ftsz(frame));
map[f++] = SNAP_MKPC(frame_contpc(frame));
frame = frame_prevd(frame);
} else {
lua_assert(!frame_isc(frame));
map[f++] = SNAP_MKFTSZ(frame_ftsz(frame));
frame = frame_prevd(frame);
continue;
}
if (frame + funcproto(frame_func(frame))->framesize > ftop)
ftop = frame + funcproto(frame_func(frame))->framesize;
}
lua_assert(f == (MSize)(1 + J->framedepth));
return (BCReg)(ftop - lim);
}
LUA_API int lua_dump(lua_State *L, lua_Writer writer, void *data)
{
cTValue *o = L->top-1;
api_check(L, L->top > L->base);
if (tvisfunc(o) && isluafunc(funcV(o)))
return lj_bcwrite(L, funcproto(funcV(o)), writer, data, 0);
else
return 1;
}
/* Return bytecode position for function/frame or NO_BCPOS. */
static BCPos debug_framepc(lua_State *L, GCfunc *fn, cTValue *nextframe)
{
const BCIns *ins;
GCproto *pt;
BCPos pos;
lua_assert(fn->c.gct == ~LJ_TFUNC || fn->c.gct == ~LJ_TTHREAD);
if (!isluafunc(fn)) { /* Cannot derive a PC for non-Lua functions. */
return NO_BCPOS;
} else if (nextframe == NULL) { /* Lua function on top. */
void *cf = cframe_raw(L->cframe);
if (cf == NULL || (char *)cframe_pc(cf) == (char *)cframe_L(cf))
return NO_BCPOS;
ins = cframe_pc(cf); /* Only happens during error/hook handling. */
} else {
if (frame_islua(nextframe)) {
ins = frame_pc(nextframe);
} else if (frame_iscont(nextframe)) {
ins = frame_contpc(nextframe);
} else {
/* Lua function below errfunc/gc/hook: find cframe to get the PC. */
void *cf = cframe_raw(L->cframe);
TValue *f = L->base-1;
for (;;) {
if (cf == NULL)
return NO_BCPOS;
while (cframe_nres(cf) < 0) {
if (f >= restorestack(L, -cframe_nres(cf)))
break;
cf = cframe_raw(cframe_prev(cf));
if (cf == NULL)
return NO_BCPOS;
}
if (f < nextframe)
break;
if (frame_islua(f)) {
f = frame_prevl(f);
} else {
if (frame_isc(f) || (LJ_HASFFI && frame_iscont(f) &&
(f-1)->u32.lo == LJ_CONT_FFI_CALLBACK))
cf = cframe_raw(cframe_prev(cf));
f = frame_prevd(f);
}
}
ins = cframe_pc(cf);
}
}
pt = funcproto(fn);
pos = proto_bcpos(pt, ins) - 1;
#if LJ_HASJIT
if (pos > pt->sizebc) { /* Undo the effects of lj_trace_exit for JLOOP. */
GCtrace *T = (GCtrace *)((char *)(ins-1) - offsetof(GCtrace, startins));
lua_assert(bc_isret(bc_op(ins[-1])));
pos = proto_bcpos(pt, mref(T->startpc, const BCIns));
}
/* Return prototype of first argument (Lua function or prototype object) */
static GCproto *check_Lproto(lua_State *L, int nolua)
{
TValue *o = L->base;
if (L->top > o) {
if (tvisproto(o)) {
return protoV(o);
} else if (tvisfunc(o)) {
if (isluafunc(funcV(o)))
return funcproto(funcV(o));
else if (nolua)
return NULL;
}
}
lj_err_argt(L, 1, LUA_TFUNCTION);
return NULL; /* unreachable */
}
/* Initialize call. Ensure stack space and return # of missing parameters. */
static int call_init(lua_State *L, GCfunc *fn)
{
if (isluafunc(fn)) {
GCproto *pt = funcproto(fn);
int numparams = pt->numparams;
int gotparams = (int)(L->top - L->base);
int need = pt->framesize;
if ((pt->flags & PROTO_IS_VARARG)) need += 1+gotparams;
lj_state_checkstack(L, (MSize)need);
numparams -= gotparams;
return numparams >= 0 ? numparams : 0;
} else {
lj_state_checkstack(L, LUA_MINSTACK);
return 0;
}
}
/* Instruction dispatch. Used by instr/line/return hooks or when recording. */
void LJ_FASTCALL lj_dispatch_ins(lua_State *L, const BCIns *pc)
{
GCfunc *fn = curr_func(L);
GCproto *pt = funcproto(fn);
void *cf = cframe_raw(L->cframe);
const BCIns *oldpc = cframe_pc(cf);
global_State *g = G(L);
BCReg slots;
setcframe_pc(cf, pc);
slots = cur_topslot(pt, pc, cframe_multres_n(cf));
L->top = L->base + slots; /* Fix top. */
#if LJ_HASJIT
{
jit_State *J = G2J(g);
if (J->state != LJ_TRACE_IDLE) {
J->L = L;
lj_trace_ins(J, pc-1); /* The interpreter bytecode PC is offset by 1. */
}
}
#endif
if ((g->hookmask & LUA_MASKCOUNT) && g->hookcount == 0) {
g->hookcount = g->hookcstart;
callhook(L, LUA_HOOKCOUNT, -1);
L->top = L->base + slots; /* Fix top again. */
}
if ((g->hookmask & LUA_MASKLINE)) {
BCPos npc = proto_bcpos(pt, pc) - 1;
BCPos opc = proto_bcpos(pt, oldpc) - 1;
BCLine line = proto_line(pt, npc);
if (pc <= oldpc || opc >= pt->sizebc || line != proto_line(pt, opc)) {
callhook(L, LUA_HOOKLINE, line);
L->top = L->base + slots; /* Fix top again. */
}
}
if ((g->hookmask & LUA_MASKRET) && bc_isret(bc_op(pc[-1])))
callhook(L, LUA_HOOKRET, -1);
}
/* Instruction dispatch callback for instr/line hooks or when recording. */
void lj_dispatch_ins(lua_State *L, const BCIns *pc, uint32_t nres)
{
GCfunc *fn = curr_func(L);
GCproto *pt = funcproto(fn);
BCReg slots = cur_topslot(pt, pc, nres);
global_State *g = G(L);
const BCIns *oldpc = cframe_Lpc(L);
cframe_Lpc(L) = pc;
L->top = L->base + slots; /* Fix top. */
#if LJ_HASJIT
{
jit_State *J = G2J(g);
if (J->state != LJ_TRACE_IDLE) {
J->L = L;
J->pc = pc-1;
J->fn = fn;
J->pt = pt;
lj_trace_ins(J);
}
}
#endif
if ((g->hookmask & LUA_MASKCOUNT) && g->hookcount == 0) {
g->hookcount = g->hookcstart;
callhook(L, LUA_HOOKCOUNT, -1);
}
if ((g->hookmask & LUA_MASKLINE) && pt->lineinfo) {
BCPos npc = (BCPos)(pc - pt->bc)-1;
BCPos opc = (BCPos)(oldpc - pt->bc)-1;
BCLine line = pt->lineinfo[npc];
if (npc == 0 || pc <= oldpc ||
opc >= pt->sizebc || line != pt->lineinfo[opc]) {
L->top = L->base + slots; /* Fix top again after instruction hook. */
callhook(L, LUA_HOOKLINE, line);
}
}
}
/* Public API function: control the JIT engine. */
int luaJIT_setmode(lua_State *L, int idx, int mode)
{
global_State *g = G(L);
int mm = mode & LUAJIT_MODE_MASK;
lj_trace_abort(g); /* Abort recording on any state change. */
/* Avoid pulling the rug from under our own feet. */
if ((g->hookmask & HOOK_GC))
lj_err_caller(L, LJ_ERR_NOGCMM);
switch (mm) {
#if LJ_HASJIT
case LUAJIT_MODE_ENGINE:
if ((mode & LUAJIT_MODE_FLUSH)) {
lj_trace_flushall(L);
} else {
if (!(mode & LUAJIT_MODE_ON))
G2J(g)->flags &= ~(uint32_t)JIT_F_ON;
#if LJ_TARGET_X86ORX64
else if ((G2J(g)->flags & JIT_F_SSE2))
G2J(g)->flags |= (uint32_t)JIT_F_ON;
else
return 0; /* Don't turn on JIT compiler without SSE2 support. */
#else
else
G2J(g)->flags |= (uint32_t)JIT_F_ON;
#endif
lj_dispatch_update(g);
}
break;
case LUAJIT_MODE_FUNC:
case LUAJIT_MODE_ALLFUNC:
case LUAJIT_MODE_ALLSUBFUNC: {
cTValue *tv = idx == 0 ? frame_prev(L->base-1) :
idx > 0 ? L->base + (idx-1) : L->top + idx;
GCproto *pt;
if ((idx == 0 || tvisfunc(tv)) && isluafunc(&gcval(tv)->fn))
pt = funcproto(&gcval(tv)->fn); /* Cannot use funcV() for frame slot. */
else if (tvisproto(tv))
pt = protoV(tv);
else
return 0; /* Failed. */
if (mm != LUAJIT_MODE_ALLSUBFUNC)
setptmode(g, pt, mode);
if (mm != LUAJIT_MODE_FUNC)
setptmode_all(g, pt, mode);
break;
}
case LUAJIT_MODE_TRACE:
if (!(mode & LUAJIT_MODE_FLUSH))
return 0; /* Failed. */
lj_trace_flush(G2J(g), idx);
break;
#else
case LUAJIT_MODE_ENGINE:
case LUAJIT_MODE_FUNC:
case LUAJIT_MODE_ALLFUNC:
case LUAJIT_MODE_ALLSUBFUNC:
UNUSED(idx);
if ((mode & LUAJIT_MODE_ON))
return 0; /* Failed. */
break;
#endif
case LUAJIT_MODE_WRAPCFUNC:
if ((mode & LUAJIT_MODE_ON)) {
if (idx != 0) {
cTValue *tv = idx > 0 ? L->base + (idx-1) : L->top + idx;
if (tvislightud(tv))
g->wrapf = (lua_CFunction)lightudV(tv);
else
return 0; /* Failed. */
} else {
return 0; /* Failed. */
}
g->bc_cfunc_ext = BCINS_AD(BC_FUNCCW, 0, 0);
} else {
g->bc_cfunc_ext = BCINS_AD(BC_FUNCC, 0, 0);
}
break;
default:
return 0; /* Failed. */
}
static GCproto *get_proto(lua_State *L) {
return funcproto(funcV(L->base));
}
/* Restore interpreter state from exit state with the help of a snapshot. */
const BCIns *lj_snap_restore(jit_State *J, void *exptr)
{
ExitState *ex = (ExitState *)exptr;
SnapNo snapno = J->exitno; /* For now, snapno == exitno. */
GCtrace *T = traceref(J, J->parent);
SnapShot *snap = &T->snap[snapno];
MSize n, nent = snap->nent;
SnapEntry *map = &T->snapmap[snap->mapofs];
SnapEntry *flinks = map + nent + snap->depth;
int32_t ftsz0;
BCReg nslots = snap->nslots;
TValue *frame;
BloomFilter rfilt = snap_renamefilter(T, snapno);
const BCIns *pc = snap_pc(map[nent]);
lua_State *L = J->L;
/* Set interpreter PC to the next PC to get correct error messages. */
setcframe_pc(cframe_raw(L->cframe), pc+1);
/* Make sure the stack is big enough for the slots from the snapshot. */
if (LJ_UNLIKELY(L->base + nslots > tvref(L->maxstack))) {
L->top = curr_topL(L);
lj_state_growstack(L, nslots - curr_proto(L)->framesize);
}
/* Fill stack slots with data from the registers and spill slots. */
frame = L->base-1;
ftsz0 = frame_ftsz(frame); /* Preserve link to previous frame in slot #0. */
for (n = 0; n < nent; n++) {
SnapEntry sn = map[n];
IRRef ref = snap_ref(sn);
BCReg s = snap_slot(sn);
TValue *o = &frame[s]; /* Stack slots are relative to start frame. */
IRIns *ir = &T->ir[ref];
if (irref_isk(ref)) { /* Restore constant slot. */
lj_ir_kvalue(L, o, ir);
if ((sn & (SNAP_CONT|SNAP_FRAME))) {
/* Overwrite tag with frame link. */
o->fr.tp.ftsz = s != 0 ? (int32_t)*flinks-- : ftsz0;
if ((sn & SNAP_FRAME)) {
GCfunc *fn = ir_kfunc(ir);
if (isluafunc(fn)) {
MSize framesize = funcproto(fn)->framesize;
L->base = ++o;
if (LJ_UNLIKELY(o + framesize > tvref(L->maxstack))) {
ptrdiff_t fsave = savestack(L, frame);
L->top = o;
lj_state_growstack(L, framesize); /* Grow again. */
frame = restorestack(L, fsave);
}
}
}
}
} else if (!(sn & SNAP_NORESTORE)) {
IRType1 t = ir->t;
RegSP rs = ir->prev;
lua_assert(!(sn & (SNAP_CONT|SNAP_FRAME)));
if (LJ_UNLIKELY(bloomtest(rfilt, ref)))
rs = snap_renameref(T, snapno, ref, rs);
if (ra_hasspill(regsp_spill(rs))) { /* Restore from spill slot. */
int32_t *sps = &ex->spill[regsp_spill(rs)];
if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) {
o->u32.lo = (uint32_t)*sps;
} else if (irt_isinteger(t)) {
setintV(o, *sps);
#if !LJ_SOFTFP
} else if (irt_isnum(t)) {
o->u64 = *(uint64_t *)sps;
#endif
#if LJ_64
} else if (irt_islightud(t)) {
/* 64 bit lightuserdata which may escape already has the tag bits. */
o->u64 = *(uint64_t *)sps;
#endif
} else {
lua_assert(!irt_ispri(t)); /* PRI refs never have a spill slot. */
setgcrefi(o->gcr, *sps);
setitype(o, irt_toitype(t));
}
} else { /* Restore from register. */
Reg r = regsp_reg(rs);
lua_assert(ra_hasreg(r));
if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) {
o->u32.lo = (uint32_t)ex->gpr[r-RID_MIN_GPR];
} else if (irt_isinteger(t)) {
setintV(o, (int32_t)ex->gpr[r-RID_MIN_GPR]);
#if !LJ_SOFTFP
} else if (irt_isnum(t)) {
setnumV(o, ex->fpr[r-RID_MIN_FPR]);
#endif
#if LJ_64
} else if (irt_islightud(t)) {
/* 64 bit lightuserdata which may escape already has the tag bits. */
o->u64 = ex->gpr[r-RID_MIN_GPR];
#endif
} else {
if (!irt_ispri(t))
setgcrefi(o->gcr, ex->gpr[r-RID_MIN_GPR]);
setitype(o, irt_toitype(t));
}
}
if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) {
rs = (ir+1)->prev;
if (LJ_UNLIKELY(bloomtest(rfilt, ref+1)))
rs = snap_renameref(T, snapno, ref+1, rs);
o->u32.hi = (ra_hasspill(regsp_spill(rs))) ?
(uint32_t)*&ex->spill[regsp_spill(rs)] :
(uint32_t)ex->gpr[regsp_reg(rs)-RID_MIN_GPR];
}
}
}
switch (bc_op(*pc)) {
case BC_CALLM: case BC_CALLMT: case BC_RETM: case BC_TSETM:
L->top = frame + nslots;
break;
default:
L->top = curr_topL(L);
break;
}
lua_assert(map + nent == flinks);
return pc;
}
/* Restore interpreter state from exit state with the help of a snapshot. */
void lj_snap_restore(jit_State *J, void *exptr)
{
ExitState *ex = (ExitState *)exptr;
SnapNo snapno = J->exitno; /* For now, snapno == exitno. */
Trace *T = J->trace[J->parent];
SnapShot *snap = &T->snap[snapno];
BCReg s, nslots = snap->nslots;
IRRef2 *map = &T->snapmap[snap->mapofs];
IRRef2 *flinks = map + nslots + snap->nframelinks;
TValue *o, *newbase, *ntop;
BloomFilter rfilt = snap_renamefilter(T, snapno);
lua_State *L = J->L;
/* Make sure the stack is big enough for the slots from the snapshot. */
if (L->base + nslots >= L->maxstack) {
L->top = curr_topL(L);
lj_state_growstack(L, nslots - curr_proto(L)->framesize);
}
/* Fill stack slots with data from the registers and spill slots. */
newbase = NULL;
ntop = L->base;
for (s = 0, o = L->base-1; s < nslots; s++, o++) {
IRRef ref = snap_ref(map[s]);
if (ref) {
IRIns *ir = &T->ir[ref];
if (irref_isk(ref)) { /* Restore constant slot. */
lj_ir_kvalue(L, o, ir);
} else {
IRType1 t = ir->t;
RegSP rs = ir->prev;
if (LJ_UNLIKELY(bloomtest(rfilt, ref)))
rs = snap_renameref(T, snapno, ref, rs);
if (ra_hasspill(regsp_spill(rs))) { /* Restore from spill slot. */
int32_t *sps = &ex->spill[regsp_spill(rs)];
if (irt_isinteger(t)) {
setintV(o, *sps);
} else if (irt_isnum(t)) {
o->u64 = *(uint64_t *)sps;
} else {
lua_assert(!irt_ispri(t)); /* PRI refs never have a spill slot. */
setgcrefi(o->gcr, *sps);
setitype(o, irt_toitype(t));
}
} else if (ra_hasreg(regsp_reg(rs))) { /* Restore from register. */
Reg r = regsp_reg(rs);
if (irt_isinteger(t)) {
setintV(o, ex->gpr[r-RID_MIN_GPR]);
} else if (irt_isnum(t)) {
setnumV(o, ex->fpr[r-RID_MIN_FPR]);
} else {
if (!irt_ispri(t))
setgcrefi(o->gcr, ex->gpr[r-RID_MIN_GPR]);
setitype(o, irt_toitype(t));
}
} else { /* Restore frame slot. */
lua_assert(ir->o == IR_FRAME);
/* This works for both PTR and FUNC IR_FRAME. */
setgcrefp(o->fr.func, mref(T->ir[ir->op2].ptr, void));
if (s != 0) /* Do not overwrite link to previous frame. */
o->fr.tp.ftsz = (int32_t)*--flinks;
if (irt_isfunc(ir->t)) {
GCfunc *fn = gco2func(gcref(T->ir[ir->op2].gcr));
if (isluafunc(fn)) {
TValue *fs;
newbase = o+1;
fs = newbase + funcproto(fn)->framesize;
if (fs > ntop) ntop = fs; /* Update top for newly added frames. */
}
}
}
}
} else if (newbase) {
setnilV(o); /* Clear unreferenced slots of newly added frames. */
}
}
