TValue * LJ_FASTCALL lj_meta_equal_cd(lua_State *L, BCIns ins)
{
ASMFunction cont = (bc_op(ins) & 1) ? lj_cont_condf : lj_cont_condt;
int op = (int)bc_op(ins) & ~1;
TValue tv;
cTValue *mo, *o2, *o1 = &L->base[bc_a(ins)];
cTValue *o1mm = o1;
if (op == BC_ISEQV) {
o2 = &L->base[bc_d(ins)];
if (!tviscdata(o1mm)) o1mm = o2;
} else if (op == BC_ISEQS) {
setstrV(L, &tv, gco2str(proto_kgc(curr_proto(L), ~(ptrdiff_t)bc_d(ins))));
o2 = &tv;
} else if (op == BC_ISEQN) {
o2 = &mref(curr_proto(L)->k, cTValue)[bc_d(ins)];
} else {
lua_assert(op == BC_ISEQP);
setpriV(&tv, ~bc_d(ins));
o2 = &tv;
}
mo = lj_meta_lookup(L, o1mm, MM_eq);
if (LJ_LIKELY(!tvisnil(mo)))
return mmcall(L, cont, mo, o1, o2);
else
return (TValue *)(intptr_t)(bc_op(ins) & 1);
}
/* Calculate number of used stack slots in the current frame. */
static BCReg cur_topslot(GCproto *pt, const BCIns *pc, uint32_t nres)
{
BCIns ins = pc[-1];
if (bc_op(ins) == BC_UCLO)
ins = pc[bc_j(ins)];
switch (bc_op(ins)) {
case BC_CALLM: case BC_CALLMT: return bc_a(ins) + bc_c(ins) + nres-1+1;
case BC_RETM: return bc_a(ins) + bc_d(ins) + nres-1;
case BC_TSETM: return bc_a(ins) + nres-1;
default: return pt->framesize;
}
}
/* Re-enable compiling a prototype by unpatching any modified bytecode. */
void lj_trace_reenableproto(GCproto *pt)
{
if ((pt->flags & PROTO_ILOOP)) {
BCIns *bc = proto_bc(pt);
BCPos i, sizebc = pt->sizebc;;
pt->flags &= ~PROTO_ILOOP;
if (bc_op(bc[0]) == BC_IFUNCF)
setbc_op(&bc[0], BC_FUNCF);
for (i = 1; i < sizebc; i++) {
BCOp op = bc_op(bc[i]);
if (op == BC_IFORL || op == BC_IITERL || op == BC_ILOOP)
setbc_op(&bc[i], (int)op+(int)BC_LOOP-(int)BC_ILOOP);
}
}
}
/**
* Called whenever the interpreter performs a non-local jump (i.e.,
* CALL[T], EVAL).
*
* Takes care of incrementing hotness counters and switching the
* interpreter mode.
*
* @return address where to continue interpreting.
*/
BCIns *
interpreterBranch(Capability *cap, JitState *J, BCIns *src_pc,
BCIns *dst_pc, Word *base, BranchType branchType)
{
if (LC_UNLIKELY(J->mode == JIT_MODE_RECORDING)) {
/* TODO: check for recording termination. */
return dst_pc;
} else {
if (dst_pc < src_pc && bc_op(*dst_pc) != BC_JFUNC) {
if (incrementHotCounter(cap, J, dst_pc) && !(cap->flags & CF_NO_JIT)) {
/* It's hot now. */
/* Start recording mode. */
Thread *T = cap->T;
cap->dispatch = cap->dispatch_record;
T->pc = dst_pc;
TraceType traceType =
branchType == BRANCH_RETURN ? RETURN_TRACE : FUNCTION_TRACE;
startRecording(J, dst_pc, T, base, traceType);
return cap->reload_state_pc;
}
}
return dst_pc;
}
}
/* Return bytecode position for function/frame or NO_BCPOS. */
static BCPos debug_framepc(lua_State *L, GCfunc *fn, cTValue *nextframe)
{
const BCIns *ins;
GCproto *pt;
BCPos pos;
lua_assert(fn->c.gct == ~LJ_TFUNC || fn->c.gct == ~LJ_TTHREAD);
if (!isluafunc(fn)) { /* Cannot derive a PC for non-Lua functions. */
return NO_BCPOS;
} else if (nextframe == NULL) { /* Lua function on top. */
void *cf = cframe_raw(L->cframe);
if (cf == NULL || (char *)cframe_pc(cf) == (char *)cframe_L(cf))
return NO_BCPOS;
ins = cframe_pc(cf); /* Only happens during error/hook handling. */
} else {
if (frame_islua(nextframe)) {
ins = frame_pc(nextframe);
} else if (frame_iscont(nextframe)) {
ins = frame_contpc(nextframe);
} else {
/* Lua function below errfunc/gc/hook: find cframe to get the PC. */
void *cf = cframe_raw(L->cframe);
TValue *f = L->base-1;
for (;;) {
if (cf == NULL)
return NO_BCPOS;
while (cframe_nres(cf) < 0) {
if (f >= restorestack(L, -cframe_nres(cf)))
break;
cf = cframe_raw(cframe_prev(cf));
if (cf == NULL)
return NO_BCPOS;
}
if (f < nextframe)
break;
if (frame_islua(f)) {
f = frame_prevl(f);
} else {
if (frame_isc(f) || (LJ_HASFFI && frame_iscont(f) &&
(f-1)->u32.lo == LJ_CONT_FFI_CALLBACK))
cf = cframe_raw(cframe_prev(cf));
f = frame_prevd(f);
}
}
ins = cframe_pc(cf);
}
}
pt = funcproto(fn);
pos = proto_bcpos(pt, ins) - 1;
#if LJ_HASJIT
if (pos > pt->sizebc) { /* Undo the effects of lj_trace_exit for JLOOP. */
GCtrace *T = (GCtrace *)((char *)(ins-1) - offsetof(GCtrace, startins));
lua_assert(bc_isret(bc_op(ins[-1])));
pos = proto_bcpos(pt, mref(T->startpc, const BCIns));
}
/* Unpatch the bytecode modified by a root trace. */
static void trace_unpatch(jit_State *J, GCtrace *T)
{
BCOp op = bc_op(T->startins);
BCIns *pc = mref(T->startpc, BCIns);
UNUSED(J);
if (op == BC_JMP)
return; /* No need to unpatch branches in parent traces (yet). */
switch (bc_op(*pc)) {
case BC_JFORL:
lua_assert(traceref(J, bc_d(*pc)) == T);
*pc = T->startins;
pc += bc_j(T->startins);
lua_assert(bc_op(*pc) == BC_JFORI);
setbc_op(pc, BC_FORI);
break;
case BC_JITERL:
case BC_JLOOP:
lua_assert(op == BC_ITERL || op == BC_LOOP || bc_isret(op));
*pc = T->startins;
break;
case BC_JMP:
lua_assert(op == BC_ITERL);
pc += bc_j(*pc)+2;
if (bc_op(*pc) == BC_JITERL) {
lua_assert(traceref(J, bc_d(*pc)) == T);
*pc = T->startins;
}
break;
case BC_JFUNCF:
lua_assert(op == BC_FUNCF);
*pc = T->startins;
break;
default: /* Already unpatched. */
break;
}
}
/* Call dispatch. Used by call hooks, hot calls or when recording. */
ASMFunction LJ_FASTCALL lj_dispatch_call(lua_State *L, const BCIns *pc)
{
GCfunc *fn = curr_func(L);
BCOp op;
global_State *g = G(L);
#if LJ_HASJIT
jit_State *J = G2J(g);
#endif
int missing = call_init(L, fn);
#if LJ_HASJIT
J->L = L;
if ((uintptr_t)pc & 1) { /* Marker for hot call. */
pc = (const BCIns *)((uintptr_t)pc & ~(uintptr_t)1);
lj_trace_hot(J, pc);
goto out;
} else if (J->state != LJ_TRACE_IDLE &&
!(g->hookmask & (HOOK_GC|HOOK_VMEVENT))) {
/* Record the FUNC* bytecodes, too. */
lj_trace_ins(J, pc-1); /* The interpreter bytecode PC is offset by 1. */
}
#endif
if ((g->hookmask & LUA_MASKCALL)) {
int i;
for (i = 0; i < missing; i++) /* Add missing parameters. */
setnilV(L->top++);
callhook(L, LUA_HOOKCALL, -1);
/* Preserve modifications of missing parameters by lua_setlocal(). */
while (missing-- > 0 && tvisnil(L->top - 1))
L->top--;
}
#if LJ_HASJIT
out:
#endif
op = bc_op(pc[-1]); /* Get FUNC* op. */
#if LJ_HASJIT
/* Use the non-hotcounting variants if JIT is off or while recording. */
if ((!(J->flags & JIT_F_ON) || J->state != LJ_TRACE_IDLE) &&
(op == BC_FUNCF || op == BC_FUNCV))
op = (BCOp)((int)op+(int)BC_IFUNCF-(int)BC_FUNCF);
#endif
return makeasmfunc(lj_bc_ofs[op]); /* Return static dispatch target. */
}
/* Instruction dispatch. Used by instr/line/return hooks or when recording. */
void LJ_FASTCALL lj_dispatch_ins(lua_State *L, const BCIns *pc)
{
GCfunc *fn = curr_func(L);
GCproto *pt = funcproto(fn);
void *cf = cframe_raw(L->cframe);
const BCIns *oldpc = cframe_pc(cf);
global_State *g = G(L);
BCReg slots;
setcframe_pc(cf, pc);
slots = cur_topslot(pt, pc, cframe_multres_n(cf));
L->top = L->base + slots; /* Fix top. */
#if LJ_HASJIT
{
jit_State *J = G2J(g);
if (J->state != LJ_TRACE_IDLE) {
J->L = L;
lj_trace_ins(J, pc-1); /* The interpreter bytecode PC is offset by 1. */
}
}
#endif
if ((g->hookmask & LUA_MASKCOUNT) && g->hookcount == 0) {
g->hookcount = g->hookcstart;
callhook(L, LUA_HOOKCOUNT, -1);
L->top = L->base + slots; /* Fix top again. */
}
if ((g->hookmask & LUA_MASKLINE)) {
BCPos npc = proto_bcpos(pt, pc) - 1;
BCPos opc = proto_bcpos(pt, oldpc) - 1;
BCLine line = proto_line(pt, npc);
if (pc <= oldpc || opc >= pt->sizebc || line != proto_line(pt, opc)) {
callhook(L, LUA_HOOKLINE, line);
L->top = L->base + slots; /* Fix top again. */
}
}
if ((g->hookmask & LUA_MASKRET) && bc_isret(bc_op(pc[-1])))
callhook(L, LUA_HOOKRET, -1);
}
/* Restore interpreter state from exit state with the help of a snapshot. */
const BCIns *lj_snap_restore(jit_State *J, void *exptr)
{
ExitState *ex = (ExitState *)exptr;
SnapNo snapno = J->exitno; /* For now, snapno == exitno. */
GCtrace *T = traceref(J, J->parent);
SnapShot *snap = &T->snap[snapno];
MSize n, nent = snap->nent;
SnapEntry *map = &T->snapmap[snap->mapofs];
SnapEntry *flinks = &T->snapmap[snap_nextofs(T, snap)-1];
ptrdiff_t ftsz0;
TValue *frame;
BloomFilter rfilt = snap_renamefilter(T, snapno);
const BCIns *pc = snap_pc(map[nent]);
lua_State *L = J->L;
/* Set interpreter PC to the next PC to get correct error messages. */
setcframe_pc(cframe_raw(L->cframe), pc+1);
/* Make sure the stack is big enough for the slots from the snapshot. */
if (LJ_UNLIKELY(L->base + snap->topslot >= tvref(L->maxstack))) {
L->top = curr_topL(L);
lj_state_growstack(L, snap->topslot - curr_proto(L)->framesize);
}
/* Fill stack slots with data from the registers and spill slots. */
frame = L->base-1;
ftsz0 = frame_ftsz(frame); /* Preserve link to previous frame in slot #0. */
for (n = 0; n < nent; n++) {
SnapEntry sn = map[n];
if (!(sn & SNAP_NORESTORE)) {
TValue *o = &frame[snap_slot(sn)];
IRRef ref = snap_ref(sn);
IRIns *ir = &T->ir[ref];
if (ir->r == RID_SUNK) {
MSize j;
for (j = 0; j < n; j++)
if (snap_ref(map[j]) == ref) { /* De-duplicate sunk allocations. */
copyTV(L, o, &frame[snap_slot(map[j])]);
goto dupslot;
}
snap_unsink(J, T, ex, snapno, rfilt, ir, o);
dupslot:
continue;
}
snap_restoreval(J, T, ex, snapno, rfilt, ref, o);
if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM) && tvisint(o)) {
TValue tmp;
snap_restoreval(J, T, ex, snapno, rfilt, ref+1, &tmp);
o->u32.hi = tmp.u32.lo;
} else if ((sn & (SNAP_CONT|SNAP_FRAME))) {
lua_assert(!LJ_FR2); /* TODO_FR2: store 64 bit PCs. */
/* Overwrite tag with frame link. */
setframe_ftsz(o, snap_slot(sn) != 0 ? (int32_t)*flinks-- : ftsz0);
L->base = o+1;
}
}
}
lua_assert(map + nent == flinks);
/* Compute current stack top. */
switch (bc_op(*pc)) {
default:
if (bc_op(*pc) < BC_FUNCF) {
L->top = curr_topL(L);
break;
}
/* fallthrough */
case BC_CALLM: case BC_CALLMT: case BC_RETM: case BC_TSETM:
L->top = frame + snap->nslots;
break;
}
return pc;
}
/* Find unused slots with reaching-definitions bytecode data-flow analysis. */
static BCReg snap_usedef(jit_State *J, uint8_t *udf,
const BCIns *pc, BCReg maxslot)
{
BCReg s;
GCobj *o;
if (maxslot == 0) return 0;
#ifdef LUAJIT_USE_VALGRIND
/* Avoid errors for harmless reads beyond maxslot. */
memset(udf, 1, SNAP_USEDEF_SLOTS);
#else
memset(udf, 1, maxslot);
#endif
/* Treat open upvalues as used. */
o = gcref(J->L->openupval);
while (o) {
if (uvval(gco2uv(o)) < J->L->base) break;
udf[uvval(gco2uv(o)) - J->L->base] = 0;
o = gcref(o->gch.nextgc);
}
#define USE_SLOT(s) udf[(s)] &= ~1
#define DEF_SLOT(s) udf[(s)] *= 3
/* Scan through following bytecode and check for uses/defs. */
lua_assert(pc >= proto_bc(J->pt) && pc < proto_bc(J->pt) + J->pt->sizebc);
for (;;) {
BCIns ins = *pc++;
BCOp op = bc_op(ins);
switch (bcmode_b(op)) {
case BCMvar: USE_SLOT(bc_b(ins)); break;
default: break;
}
switch (bcmode_c(op)) {
case BCMvar: USE_SLOT(bc_c(ins)); break;
case BCMrbase:
lua_assert(op == BC_CAT);
for (s = bc_b(ins); s <= bc_c(ins); s++) USE_SLOT(s);
for (; s < maxslot; s++) DEF_SLOT(s);
break;
case BCMjump:
handle_jump: {
BCReg minslot = bc_a(ins);
if (op >= BC_FORI && op <= BC_JFORL) minslot += FORL_EXT;
else if (op >= BC_ITERL && op <= BC_JITERL) minslot += bc_b(pc[-2])-1;
else if (op == BC_UCLO) { pc += bc_j(ins); break; }
for (s = minslot; s < maxslot; s++) DEF_SLOT(s);
return minslot < maxslot ? minslot : maxslot;
}
case BCMlit:
if (op == BC_JFORL || op == BC_JITERL || op == BC_JLOOP) {
goto handle_jump;
} else if (bc_isret(op)) {
BCReg top = op == BC_RETM ? maxslot : (bc_a(ins) + bc_d(ins)-1);
for (s = 0; s < bc_a(ins); s++) DEF_SLOT(s);
for (; s < top; s++) USE_SLOT(s);
for (; s < maxslot; s++) DEF_SLOT(s);
return 0;
}
break;
case BCMfunc: return maxslot; /* NYI: will abort, anyway. */
default: break;
}
switch (bcmode_a(op)) {
case BCMvar: USE_SLOT(bc_a(ins)); break;
case BCMdst:
if (!(op == BC_ISTC || op == BC_ISFC)) DEF_SLOT(bc_a(ins));
break;
case BCMbase:
if (op >= BC_CALLM && op <= BC_VARG) {
BCReg top = (op == BC_CALLM || op == BC_CALLMT || bc_c(ins) == 0) ?
maxslot : (bc_a(ins) + bc_c(ins)+LJ_FR2);
if (LJ_FR2) DEF_SLOT(bc_a(ins)+1);
s = bc_a(ins) - ((op == BC_ITERC || op == BC_ITERN) ? 3 : 0);
for (; s < top; s++) USE_SLOT(s);
for (; s < maxslot; s++) DEF_SLOT(s);
if (op == BC_CALLT || op == BC_CALLMT) {
for (s = 0; s < bc_a(ins); s++) DEF_SLOT(s);
return 0;
}
} else if (op == BC_KNIL) {
for (s = bc_a(ins); s <= bc_d(ins); s++) DEF_SLOT(s);
} else if (op == BC_TSETM) {
for (s = bc_a(ins)-1; s < maxslot; s++) USE_SLOT(s);
}
break;
default: break;
}
lua_assert(pc >= proto_bc(J->pt) && pc < proto_bc(J->pt) + J->pt->sizebc);
}
#undef USE_SLOT
#undef DEF_SLOT
return 0; /* unreachable */
}
u4
printInstruction_aux(FILE *stream, const BCIns *ins /*in*/, int oneline)
{
const BCIns *ins0 = ins;
u4 j;
const BCIns i = *ins;
const char *name = ins_name[bc_op(i)];
fprintf(stream, "%p: ", ins);
++ ins;
switch(ins_format[bc_op(i)]) {
case IFM_R:
fprintf(stream, "%s\tr%d\n", name, bc_a(i)); break;
case IFM_RR:
fprintf(stream, "%s\tr%d, r%d\n", name, bc_a(i), bc_d(i)); break;
case IFM_RRR:
fprintf(stream, "%s\tr%d, r%d, r%d\n", name, bc_a(i), bc_b(i), bc_c(i));
break;
case IFM_RN:
fprintf(stream, "%s\tr%d, %d\n", name, bc_a(i), bc_d(i)); break;
case IFM_RRN:
fprintf(stream, "%s\tr%d, r%d, %d\n", name, bc_a(i), bc_b(i), bc_c(i));
break;
case IFM_RS:
fprintf(stream, "%s\tr%d, %d\n", name, bc_a(i), bc_sd(i)); break;
case IFM_J:
fprintf(stream, "%s\t%p\n", name, ins + bc_j(i)); break;
case IFM_RRJ:
fprintf(stream, "%s\tr%d, r%d, %p\n", name, bc_a(i), bc_d(i),
ins + 1 + bc_j(*ins));
ins++;
break;
case IFM____:
switch (bc_op(i)) {
case BC_EVAL:
{ fprintf(stream, "EVAL\tr%d", bc_a(i));
printInlineBitmap(stream, ins);
ins++;
}
break;
case BC_CASE:
{ u2 *tgt = (u2*)ins; u4 ncases = bc_d(i);
ins += (ncases + 1) / 2;
fprintf(stream, "CASE\tr%d\n", bc_a(i));
if (!oneline) {
for (j = 0; j < ncases; j++, tgt++) {
fprintf(stream, " %d: %p\n", j + 1, ins + bc_j_from_d(*tgt));
}
}
}
break;
case BC_CASE_S:
fprintf(stream, "CASE_S\tr%d ...TODO...\n", bc_a(i));
ins += bc_d(i);
break;
case BC_ALLOC1:
fprintf(stream, "ALLOC1\tr%d, r%d, r%d", bc_a(i), bc_b(i), bc_c(i));
printInlineBitmap(stream, ins);
ins += 1;
break;
case BC_ALLOC:
{
u1 *arg = (u1*)ins; ins += 1 + BC_ROUND(bc_c(i));
fprintf(stream, "ALLOC\tr%d, r%d", bc_a(i), bc_b(i));
for (j = 0; j < bc_c(i); j++, arg++)
fprintf(stream, ", r%d", *arg);
printInlineBitmap(stream, ins - 1);
}
break;
case BC_ALLOCAP:
{
u1 *arg = (u1*)ins; ins += 1 + BC_ROUND(bc_c(i) + 1);
fprintf(stream, "ALLOCAP\tr%d", bc_a(i));
u1 ptrmask = bc_b(i);
fprintf(stream, ", r%d", *arg++);
for (j = 1; j < bc_c(i); j++, arg++) {
fprintf(stream, ", r%d%c", *arg, ptrmask & 1 ? '*' : ' ');
ptrmask >>= 1;
}
printInlineBitmap(stream, ins - 1);
}
break;
case BC_CALL:
{ u1 *arg = (u1*)ins; ins += BC_ROUND(bc_c(i)) + 1;
fprintf(stream, "%s\tr%d", name, bc_a(i));
char comma = '(';
for (j = 0; j < bc_c(i); j++, arg++) {
fprintf(stream, "%cr%d", comma, *arg);
comma = ',';
}
fprintf(stream, ") [%x]", bc_b(i));
printInlineBitmap(stream, ins - 1);
}
break;
case BC_CALLT:
{
int j;
u1 bitmask = bc_b(i);
fprintf(stream, "CALLT r%d", bc_a(i));
for (j = 0; j < bc_c(i); j++) {
fprintf(stream, "%cr%d%c", j == 0 ? '(' : ',', j, bitmask & 1 ? '*' : ' ');
bitmask >>= 1;
}
fprintf(stream, ")\n");
}
break;
default:
fprintf(stream, "%s ...TODO...\n", name);
}
break;
default:
fprintf(stderr, "FATAL: Unknown intruction format: %d\n",
ins_format[bc_op(i)]);
exit(1);
}
return (u4)(ins - ins0);
}
/* Restore interpreter state from exit state with the help of a snapshot. */
const BCIns *lj_snap_restore(jit_State *J, void *exptr)
{
ExitState *ex = (ExitState *)exptr;
SnapNo snapno = J->exitno; /* For now, snapno == exitno. */
GCtrace *T = traceref(J, J->parent);
SnapShot *snap = &T->snap[snapno];
MSize n, nent = snap->nent;
SnapEntry *map = &T->snapmap[snap->mapofs];
SnapEntry *flinks = map + nent + snap->depth;
int32_t ftsz0;
BCReg nslots = snap->nslots;
TValue *frame;
BloomFilter rfilt = snap_renamefilter(T, snapno);
const BCIns *pc = snap_pc(map[nent]);
lua_State *L = J->L;
/* Set interpreter PC to the next PC to get correct error messages. */
setcframe_pc(cframe_raw(L->cframe), pc+1);
/* Make sure the stack is big enough for the slots from the snapshot. */
if (LJ_UNLIKELY(L->base + nslots > tvref(L->maxstack))) {
L->top = curr_topL(L);
lj_state_growstack(L, nslots - curr_proto(L)->framesize);
}
/* Fill stack slots with data from the registers and spill slots. */
frame = L->base-1;
ftsz0 = frame_ftsz(frame); /* Preserve link to previous frame in slot #0. */
for (n = 0; n < nent; n++) {
SnapEntry sn = map[n];
IRRef ref = snap_ref(sn);
BCReg s = snap_slot(sn);
TValue *o = &frame[s]; /* Stack slots are relative to start frame. */
IRIns *ir = &T->ir[ref];
if (irref_isk(ref)) { /* Restore constant slot. */
lj_ir_kvalue(L, o, ir);
if ((sn & (SNAP_CONT|SNAP_FRAME))) {
/* Overwrite tag with frame link. */
o->fr.tp.ftsz = s != 0 ? (int32_t)*flinks-- : ftsz0;
if ((sn & SNAP_FRAME)) {
GCfunc *fn = ir_kfunc(ir);
if (isluafunc(fn)) {
MSize framesize = funcproto(fn)->framesize;
L->base = ++o;
if (LJ_UNLIKELY(o + framesize > tvref(L->maxstack))) {
ptrdiff_t fsave = savestack(L, frame);
L->top = o;
lj_state_growstack(L, framesize); /* Grow again. */
frame = restorestack(L, fsave);
}
}
}
}
} else if (!(sn & SNAP_NORESTORE)) {
IRType1 t = ir->t;
RegSP rs = ir->prev;
lua_assert(!(sn & (SNAP_CONT|SNAP_FRAME)));
if (LJ_UNLIKELY(bloomtest(rfilt, ref)))
rs = snap_renameref(T, snapno, ref, rs);
if (ra_hasspill(regsp_spill(rs))) { /* Restore from spill slot. */
int32_t *sps = &ex->spill[regsp_spill(rs)];
if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) {
o->u32.lo = (uint32_t)*sps;
} else if (irt_isinteger(t)) {
setintV(o, *sps);
#if !LJ_SOFTFP
} else if (irt_isnum(t)) {
o->u64 = *(uint64_t *)sps;
#endif
#if LJ_64
} else if (irt_islightud(t)) {
/* 64 bit lightuserdata which may escape already has the tag bits. */
o->u64 = *(uint64_t *)sps;
#endif
} else {
lua_assert(!irt_ispri(t)); /* PRI refs never have a spill slot. */
setgcrefi(o->gcr, *sps);
setitype(o, irt_toitype(t));
}
} else { /* Restore from register. */
Reg r = regsp_reg(rs);
lua_assert(ra_hasreg(r));
if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) {
o->u32.lo = (uint32_t)ex->gpr[r-RID_MIN_GPR];
} else if (irt_isinteger(t)) {
setintV(o, (int32_t)ex->gpr[r-RID_MIN_GPR]);
#if !LJ_SOFTFP
} else if (irt_isnum(t)) {
setnumV(o, ex->fpr[r-RID_MIN_FPR]);
#endif
#if LJ_64
} else if (irt_islightud(t)) {
/* 64 bit lightuserdata which may escape already has the tag bits. */
o->u64 = ex->gpr[r-RID_MIN_GPR];
#endif
} else {
if (!irt_ispri(t))
setgcrefi(o->gcr, ex->gpr[r-RID_MIN_GPR]);
setitype(o, irt_toitype(t));
}
}
if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) {
rs = (ir+1)->prev;
if (LJ_UNLIKELY(bloomtest(rfilt, ref+1)))
rs = snap_renameref(T, snapno, ref+1, rs);
o->u32.hi = (ra_hasspill(regsp_spill(rs))) ?
(uint32_t)*&ex->spill[regsp_spill(rs)] :
(uint32_t)ex->gpr[regsp_reg(rs)-RID_MIN_GPR];
}
}
}
switch (bc_op(*pc)) {
case BC_CALLM: case BC_CALLMT: case BC_RETM: case BC_TSETM:
L->top = frame + nslots;
break;
default:
L->top = curr_topL(L);
break;
}
lua_assert(map + nent == flinks);
return pc;
}
