/* Set/reset protection to allow patching of MCode areas. */
MCode *lj_mcode_patch(jit_State *J, MCode *ptr, int finish)
{
#ifdef LUAJIT_UNPROTECT_MCODE
UNUSED(J); UNUSED(ptr); UNUSED(finish);
return NULL;
#else
if (finish) {
if (J->mcarea == ptr)
mcode_protect(J, MCPROT_RUN);
else if (LJ_UNLIKELY(mcode_setprot(ptr, ((MCLink *)ptr)->size, MCPROT_RUN)))
mcode_protfail(J);
return NULL;
} else {
MCode *mc = J->mcarea;
/* Try current area first to use the protection cache. */
if (ptr >= mc && ptr < (MCode *)((char *)mc + J->szmcarea)) {
mcode_protect(J, MCPROT_GEN);
return mc;
}
/* Otherwise search through the list of MCode areas. */
for (;;) {
mc = ((MCLink *)mc)->next;
lua_assert(mc != NULL);
if (ptr >= mc && ptr < (MCode *)((char *)mc + ((MCLink *)mc)->size)) {
if (LJ_UNLIKELY(mcode_setprot(mc, ((MCLink *)mc)->size, MCPROT_GEN)))
mcode_protfail(J);
return mc;
}
}
}
#endif
}
/* Loop optimization. */
int lj_opt_loop(jit_State *J)
{
IRRef nins = J->cur.nins;
SnapNo nsnap = J->cur.nsnap;
MSize nsnapmap = J->cur.nsnapmap;
int errcode = lj_vm_cpcall(J->L, NULL, J, cploop_opt);
if (LJ_UNLIKELY(errcode)) {
lua_State *L = J->L;
if (errcode == LUA_ERRRUN && tvisnumber(L->top-1)) { /* Trace error? */
int32_t e = numberVint(L->top-1);
switch ((TraceError)e) {
case LJ_TRERR_TYPEINS: /* Type instability. */
case LJ_TRERR_GFAIL: /* Guard would always fail. */
/* Unrolling via recording fixes many cases, e.g. a flipped boolean. */
if (--J->instunroll < 0) /* But do not unroll forever. */
break;
L->top--; /* Remove error object. */
loop_undo(J, nins, nsnap, nsnapmap);
return 1; /* Loop optimization failed, continue recording. */
default:
break;
}
}
lj_err_throw(L, errcode); /* Propagate all other errors. */
}
return 0; /* Loop optimization is ok. */
}
/* Final conversion to double. */
static void strscan_double(uint64_t x, tl_value *o, int32_t ex2, int32_t neg)
{
double n;
/* Avoid double rounding for denormals. */
if (LJ_UNLIKELY(ex2 <= -1075 && x != 0)) {
/* NYI: all of this generates way too much code on 32 bit CPUs. */
#if defined(__GNUC__) && LJ_64
int32_t b = (int32_t)(__builtin_clzll(x)^63);
#else
int32_t b = tl_fls64(x);
#endif
if ((int32_t)b + ex2 <= -1023 && (int32_t)b + ex2 >= -1075) {
uint64_t rb = (uint64_t)1 << (-1075-ex2);
if ((x & rb) && ((x & (rb+rb+rb-1)))) x += rb+rb;
x = (x & ~(rb+rb-1));
}
}
/* Convert to double using a signed int64_t conversion, then rescale. */
lua_assert((int64_t)x >= 0);
n = (double)(int64_t)x;
if (neg) n = -n;
if (ex2) n = ldexp(n, ex2);
o->n = n;
}
LUA_API void lua_pushunsigned (lua_State *L, lua_Unsigned u) {
lua_Number n=0;
lua_lock(L);
/* printf("u=%d,%u,%x\n",u,u,u);
printf("INT_MAX=%d %u %x, (lua_Unsigned)INT_MAX=%d %u %x (unsigned int)INT_MAX=%d %u %x\n",
INT_MAX,INT_MAX,INT_MAX,
(lua_Unsigned)INT_MAX,(lua_Unsigned)INT_MAX,(lua_Unsigned)INT_MAX,
(unsigned int)INT_MAX,(unsigned int)INT_MAX,(unsigned int)INT_MAX);
printf("((u) <= (lua_Unsigned)INT_MAX) :%d\n",(((unsigned int)u) <= (lua_Unsigned)INT_MAX));
printf("(lua_Number)(int)(u) =%d,%u,%x\n",(lua_Number)(int)(u),(lua_Number)(int)(u),(lua_Number)(int)(u));
printf("(lua_Number)(u) =%f,%u,%x\n",(lua_Number)(u),(lua_Number)(u),(lua_Number)(u));
printf("u=%d,%u,%x\n",u,u,u);
*/
/*printf("n=%d,%u,%x u=%d,%u,%x\n",n,n,n,u,u,u); */
n = lua_unsigned2number(u);
/* printf("n=%f \n",n);
printf("u=%d %u %x n=%d %u %x\n",u,u,u,n,n,n);
printf("n=%d %u %x\n",n,n,n);
printf("u=%d,%u,%x\n",u,u,u);
*/
/* setintptrV(L->top, n);
incr_top(L);
setnumV(L->top, n);
*/
setnumV(L->top, n);
if (LJ_UNLIKELY(tvisnan(L->top)))
setnanV(L->top); /* Canonicalize injected NaNs. */
incr_top(L);
/*setnvalue(L->top, n);
api_incr_top(L);
*/
lua_unlock(L);
}
/* Take a snapshot of the current stack. */
static void snapshot_stack(jit_State *J, SnapShot *snap, MSize nsnapmap)
{
BCReg nslots = J->baseslot + J->maxslot;
MSize nsm, nframelinks;
IRRef2 *p;
/* Conservative estimate. Continuation frames need 2 slots. */
nsm = nsnapmap + nslots + (uint32_t)J->framedepth*2+1;
if (LJ_UNLIKELY(nsm > J->sizesnapmap)) { /* Need to grow snapshot map? */
if (nsm < 2*J->sizesnapmap)
nsm = 2*J->sizesnapmap;
else if (nsm < 64)
nsm = 64;
J->snapmapbuf = (IRRef2 *)lj_mem_realloc(J->L, J->snapmapbuf,
J->sizesnapmap*sizeof(IRRef2), nsm*sizeof(IRRef2));
J->cur.snapmap = J->snapmapbuf;
J->sizesnapmap = nsm;
}
p = &J->cur.snapmap[nsnapmap];
snapshot_slots(J, p, nslots);
nframelinks = snapshot_framelinks(J, p + nslots);
J->cur.nsnapmap = (uint16_t)(nsnapmap + nslots + nframelinks);
snap->mapofs = (uint16_t)nsnapmap;
snap->ref = (IRRef1)J->cur.nins;
snap->nslots = (uint8_t)nslots;
snap->nframelinks = (uint8_t)nframelinks;
snap->count = 0;
}
static LJ_AINLINE void save(LexState *ls, int c)
{
if (LJ_UNLIKELY(ls->sb.n + 1 > ls->sb.sz))
save_grow(ls, c);
else
ls->sb.buf[ls->sb.n++] = (char)c;
}
LJ_NOINLINE char *LJ_FASTCALL lj_buf_need2(SBuf *sb, MSize sz)
{
lua_assert(sz > sbufsz(sb));
if (LJ_UNLIKELY(sz > LJ_MAX_MEM))
lj_err_mem(sbufL(sb));
buf_grow(sb, sz);
return sbufB(sb);
}
/* Change protection of MCode area. */
static void mcode_protect(jit_State *J, int prot)
{
if (J->mcprot != prot) {
if (LJ_UNLIKELY(mcode_setprot(J->mcarea, J->szmcarea, prot)))
mcode_protfail(J);
J->mcprot = prot;
}
}
/* Restore raw data from the trace exit state. */
static void snap_restoredata(GCtrace *T, ExitState *ex,
SnapNo snapno, BloomFilter rfilt,
IRRef ref, void *dst, CTSize sz)
{
IRIns *ir = &T->ir[ref];
RegSP rs = ir->prev;
int32_t *src;
uint64_t tmp;
if (irref_isk(ref)) {
if (ir->o == IR_KNUM || ir->o == IR_KINT64) {
src = mref(ir->ptr, int32_t);
} else if (sz == 8) {
tmp = (uint64_t)(uint32_t)ir->i;
src = (int32_t *)&tmp;
} else {
src = &ir->i;
}
} else {
if (LJ_UNLIKELY(bloomtest(rfilt, ref)))
rs = snap_renameref(T, snapno, ref, rs);
if (ra_hasspill(regsp_spill(rs))) {
src = &ex->spill[regsp_spill(rs)];
if (sz == 8 && !irt_is64(ir->t)) {
tmp = (uint64_t)(uint32_t)*src;
src = (int32_t *)&tmp;
}
} else {
Reg r = regsp_reg(rs);
if (ra_noreg(r)) {
/* Note: this assumes CNEWI is never used for SOFTFP split numbers. */
lua_assert(sz == 8 && ir->o == IR_CONV && ir->op2 == IRCONV_NUM_INT);
snap_restoredata(T, ex, snapno, rfilt, ir->op1, dst, 4);
*(lua_Number *)dst = (lua_Number)*(int32_t *)dst;
return;
}
src = (int32_t *)&ex->gpr[r-RID_MIN_GPR];
#if !LJ_SOFTFP
if (r >= RID_MAX_GPR) {
src = (int32_t *)&ex->fpr[r-RID_MIN_FPR];
#if LJ_TARGET_PPC
if (sz == 4) { /* PPC FPRs are always doubles. */
*(float *)dst = (float)*(double *)src;
return;
}
#else
if (LJ_BE && sz == 4) src++;
#endif
}
#endif
}
}
lua_assert(sz == 1 || sz == 2 || sz == 4 || sz == 8);
if (sz == 4) *(int32_t *)dst = *src;
else if (sz == 8) *(int64_t *)dst = *(int64_t *)src;
else if (sz == 1) *(int8_t *)dst = (int8_t)*src;
else *(int16_t *)dst = (int16_t)*src;
}
LJ_NOINLINE char *LJ_FASTCALL lj_buf_more2(SBuf *sb, MSize sz)
{
MSize len = sbuflen(sb);
lua_assert(sz > sbufleft(sb));
if (LJ_UNLIKELY(sz > LJ_MAX_MEM || len + sz > LJ_MAX_MEM))
lj_err_mem(sbufL(sb));
buf_grow(sb, len + sz);
return sbufP(sb);
}
/* Restore a value from the trace exit state. */
static void snap_restoreval(jit_State *J, GCtrace *T, ExitState *ex,
SnapNo snapno, BloomFilter rfilt,
IRRef ref, TValue *o)
{
IRIns *ir = &T->ir[ref];
IRType1 t = ir->t;
RegSP rs = ir->prev;
if (irref_isk(ref)) { /* Restore constant slot. */
lj_ir_kvalue(J->L, o, ir);
return;
}
if (LJ_UNLIKELY(bloomtest(rfilt, ref)))
rs = snap_renameref(T, snapno, ref, rs);
lua_assert(!LJ_GC64); /* TODO_GC64: handle 64 bit references. */
if (ra_hasspill(regsp_spill(rs))) { /* Restore from spill slot. */
int32_t *sps = &ex->spill[regsp_spill(rs)];
if (irt_isinteger(t)) {
setintV(o, *sps);
#if !LJ_SOFTFP
} else if (irt_isnum(t)) {
o->u64 = *(uint64_t *)sps;
#endif
} else if (LJ_64 && irt_islightud(t)) {
/* 64 bit lightuserdata which may escape already has the tag bits. */
o->u64 = *(uint64_t *)sps;
} else {
lua_assert(!irt_ispri(t)); /* PRI refs never have a spill slot. */
setgcV(J->L, o, (GCobj *)(uintptr_t)*(GCSize *)sps, irt_toitype(t));
}
} else { /* Restore from register. */
Reg r = regsp_reg(rs);
if (ra_noreg(r)) {
lua_assert(ir->o == IR_CONV && ir->op2 == IRCONV_NUM_INT);
snap_restoreval(J, T, ex, snapno, rfilt, ir->op1, o);
if (LJ_DUALNUM) setnumV(o, (lua_Number)intV(o));
return;
} else if (irt_isinteger(t)) {
setintV(o, (int32_t)ex->gpr[r-RID_MIN_GPR]);
#if !LJ_SOFTFP
} else if (irt_isnum(t)) {
setnumV(o, ex->fpr[r-RID_MIN_FPR]);
#endif
} else if (LJ_64 && irt_is64(t)) {
/* 64 bit values that already have the tag bits. */
o->u64 = ex->gpr[r-RID_MIN_GPR];
} else if (irt_ispri(t)) {
setpriV(o, irt_toitype(t));
} else {
setgcV(J->L, o, (GCobj *)ex->gpr[r-RID_MIN_GPR], irt_toitype(t));
}
}
}
/* Index a C library by name. */
TValue *lj_clib_index(lua_State *L, CLibrary *cl, GCstr *name)
{
TValue *tv = lj_tab_setstr(L, cl->cache, name);
if (LJ_UNLIKELY(tvisnil(tv))) {
CTState *cts = ctype_cts(L);
CType *ct;
CTypeID id = lj_ctype_getname(cts, &ct, name, CLNS_INDEX);
if (!id)
lj_err_callerv(L, LJ_ERR_FFI_NODECL, strdata(name));
if (ctype_isconstval(ct->info)) {
CType *ctt = ctype_child(cts, ct);
lua_assert(ctype_isinteger(ctt->info) && ctt->size <= 4);
if ((ctt->info & CTF_UNSIGNED) && (int32_t)ct->size < 0)
setnumV(tv, (lua_Number)(uint32_t)ct->size);
else
setintV(tv, (int32_t)ct->size);
} else {
const char *sym = clib_extsym(cts, ct, name);
#if LJ_TARGET_WINDOWS
DWORD oldwerr = GetLastError();
#endif
void *p = clib_getsym(cl, sym);
GCcdata *cd;
lua_assert(ctype_isfunc(ct->info) || ctype_isextern(ct->info));
#if LJ_TARGET_X86 && LJ_ABI_WIN
/* Retry with decorated name for fastcall/stdcall functions. */
if (!p && ctype_isfunc(ct->info)) {
CTInfo cconv = ctype_cconv(ct->info);
if (cconv == CTCC_FASTCALL || cconv == CTCC_STDCALL) {
CTSize sz = clib_func_argsize(cts, ct);
const char *symd = lj_str_pushf(L,
cconv == CTCC_FASTCALL ? "@%s@%d" : "_%s@%d",
sym, sz);
L->top--;
p = clib_getsym(cl, symd);
}
}
#endif
if (!p)
clib_error(L, "cannot resolve symbol " LUA_QS ": %s", sym);
#if LJ_TARGET_WINDOWS
SetLastError(oldwerr);
#endif
cd = lj_cdata_new(cts, id, CTSIZE_PTR);
*(void **)cdataptr(cd) = p;
setcdataV(L, tv, cd);
}
}
return tv;
}
lua_Number lj_lib_checknum(lua_State *L, int narg)
{
TValue *o = L->base + narg-1;
if (!(o < L->top &&
(tvisnumber(o) || (tvisstr(o) && lj_strscan_num(strV(o), o)))))
lj_err_argt(L, narg, LUA_TNUMBER);
if (LJ_UNLIKELY(tvisint(o))) {
lua_Number n = (lua_Number)intV(o);
setnumV(o, n);
return n;
} else {
return numV(o);
}
}
/* Helper for ordered comparisons. String compare, __lt/__le metamethods. */
TValue *lj_meta_comp(lua_State *L, cTValue *o1, cTValue *o2, int op)
{
if (LJ_HASFFI && (tviscdata(o1) || tviscdata(o2))) {
ASMFunction cont = (op & 1) ? lj_cont_condf : lj_cont_condt;
MMS mm = (op & 2) ? MM_le : MM_lt;
cTValue *mo = lj_meta_lookup(L, tviscdata(o1) ? o1 : o2, mm);
if (LJ_UNLIKELY(tvisnil(mo))) goto err;
return mmcall(L, cont, mo, o1, o2);
} else if (LJ_52 || itype(o1) == itype(o2)) {
/* Never called with two numbers. */
if (tvisstr(o1) && tvisstr(o2)) {
int32_t res = lj_str_cmp(strV(o1), strV(o2));
return (TValue *)(intptr_t)(((op&2) ? res <= 0 : res < 0) ^ (op&1));
} else {
trymt:
while (1) {
ASMFunction cont = (op & 1) ? lj_cont_condf : lj_cont_condt;
MMS mm = (op & 2) ? MM_le : MM_lt;
cTValue *mo = lj_meta_lookup(L, o1, mm);
#if LJ_52
if (tvisnil(mo) && tvisnil((mo = lj_meta_lookup(L, o2, mm))))
#else
cTValue *mo2 = lj_meta_lookup(L, o2, mm);
if (tvisnil(mo) || !lj_obj_equal(mo, mo2))
#endif
{
if (op & 2) { /* MM_le not found: retry with MM_lt. */
cTValue *ot = o1; o1 = o2; o2 = ot; /* Swap operands. */
op ^= 3; /* Use LT and flip condition. */
continue;
}
goto err;
}
return mmcall(L, cont, mo, o1, o2);
}
}
} else if (tvisbool(o1) && tvisbool(o2)) {
goto trymt;
} else {
err:
lj_err_comp(L, o1, o2);
return NULL;
}
}
void lj_vmevent_call(lua_State *L, ptrdiff_t argbase, int keeponstack)
{
global_State *g = G(L);
uint8_t oldmask = g->vmevmask;
uint8_t oldh = hook_save(g);
int status;
g->vmevmask = 0; /* Disable all events. */
hook_vmevent(g);
status = lj_vm_pcall(L, restorestack(L, argbase), 0+1+keeponstack, 0);
if (LJ_UNLIKELY(status)) {
/* Really shouldn't use stderr here, but where else to complain? */
L->top--;
fprintf(stderr, "VM handler failed: %s\n",
tvisstr(L->top) ? strVdata(L->top) : "?");
}
hook_restore(g, oldh);
if (g->vmevmask != VMEVENT_NOCACHE)
g->vmevmask = oldmask; /* Restore event mask, but not if not modified. */
}
/* Add or merge a snapshot. */
void lj_snap_add(jit_State *J)
{
MSize nsnap = J->cur.nsnap;
MSize nsnapmap = J->cur.nsnapmap;
/* Merge if no ins. inbetween or if requested and no guard inbetween. */
if (J->mergesnap ? !irt_isguard(J->guardemit) :
(nsnap > 0 && J->cur.snap[nsnap-1].ref == J->cur.nins)) {
nsnapmap = J->cur.snap[--nsnap].mapofs;
} else {
/* Need to grow snapshot buffer? */
if (LJ_UNLIKELY(nsnap >= J->sizesnap)) {
MSize maxsnap = (MSize)J->param[JIT_P_maxsnap];
if (nsnap >= maxsnap)
lj_trace_err(J, LJ_TRERR_SNAPOV);
lj_mem_growvec(J->L, J->snapbuf, J->sizesnap, maxsnap, SnapShot);
J->cur.snap = J->snapbuf;
}
J->cur.nsnap = (uint16_t)(nsnap+1);
}
J->mergesnap = 0;
J->guardemit.irt = 0;
snapshot_stack(J, &J->cur.snap[nsnap], nsnapmap);
}
/* Free a C data object. */
void LJ_FASTCALL lj_cdata_free(global_State *g, GCcdata *cd)
{
if (LJ_UNLIKELY(cd->marked & LJ_GC_CDATA_FIN)) {
GCobj *root;
makewhite(g, obj2gco(cd));
markfinalized(obj2gco(cd));
if ((root = gcref(g->gc.mmudata)) != NULL) {
setgcrefr(cd->nextgc, root->gch.nextgc);
setgcref(root->gch.nextgc, obj2gco(cd));
setgcref(g->gc.mmudata, obj2gco(cd));
} else {
setgcref(cd->nextgc, obj2gco(cd));
setgcref(g->gc.mmudata, obj2gco(cd));
}
} else if (LJ_LIKELY(!cdataisv(cd))) {
CType *ct = ctype_raw(ctype_ctsG(g), cd->ctypeid);
CTSize sz = ctype_hassize(ct->info) ? ct->size : CTSIZE_PTR;
lua_assert(ctype_hassize(ct->info) || ctype_isfunc(ct->info) ||
ctype_isextern(ct->info));
lj_mem_free(g, cd, sizeof(GCcdata) + sz);
} else {
lj_mem_free(g, memcdatav(cd), sizecdatav(cd));
}
}
/* Helper for CAT. Coercion, iterative concat, __concat metamethod. */
TValue *lj_meta_cat(lua_State *L, TValue *top, int left)
{
int fromc = 0;
if (left < 0) { left = -left; fromc = 1; }
do {
int n = 1;
if (!(tvisstr(top-1) || tvisnumber(top-1)) || !tostring(L, top)) {
cTValue *mo = lj_meta_lookup(L, top-1, MM_concat);
if (tvisnil(mo)) {
mo = lj_meta_lookup(L, top, MM_concat);
if (tvisnil(mo)) {
if (tvisstr(top-1) || tvisnumber(top-1)) top++;
lj_err_optype(L, top-1, LJ_ERR_OPCAT);
return NULL; /* unreachable */
}
}
/* One of the top two elements is not a string, call __cat metamethod:
**
** before: [...][CAT stack .........................]
** top-1 top top+1 top+2
** pick two: [...][CAT stack ...] [o1] [o2]
** setup mm: [...][CAT stack ...] [cont|?] [mo|tmtype] [o1] [o2]
** in asm: [...][CAT stack ...] [cont|PC] [mo|delta] [o1] [o2]
** ^-- func base ^-- mm base
** after mm: [...][CAT stack ...] <--push-- [result]
** next step: [...][CAT stack .............]
*/
copyTV(L, top+2, top); /* Careful with the order of stack copies! */
copyTV(L, top+1, top-1);
copyTV(L, top, mo);
setcont(top-1, lj_cont_cat);
return top+1; /* Trigger metamethod call. */
} else if (strV(top)->len == 0) { /* Shortcut. */
(void)tostring(L, top-1);
} else {
/* Pick as many strings as possible from the top and concatenate them:
**
** before: [...][CAT stack ...........................]
** pick str: [...][CAT stack ...] [...... strings ......]
** concat: [...][CAT stack ...] [result]
** next step: [...][CAT stack ............]
*/
MSize tlen = strV(top)->len;
char *buffer;
int i;
for (n = 1; n <= left && tostring(L, top-n); n++) {
MSize len = strV(top-n)->len;
if (len >= LJ_MAX_STR - tlen)
lj_err_msg(L, LJ_ERR_STROV);
tlen += len;
}
buffer = lj_str_needbuf(L, &G(L)->tmpbuf, tlen);
n--;
tlen = 0;
for (i = n; i >= 0; i--) {
MSize len = strV(top-i)->len;
memcpy(buffer + tlen, strVdata(top-i), len);
tlen += len;
}
setstrV(L, top-n, lj_str_new(L, buffer, tlen));
}
left -= n;
top -= n;
} while (left >= 1);
if (LJ_UNLIKELY(G(L)->gc.total >= G(L)->gc.threshold)) {
if (!fromc) L->top = curr_topL(L);
lj_gc_step(L);
}
return NULL;
}
/* Restore interpreter state from exit state with the help of a snapshot. */
void lj_snap_restore(jit_State *J, void *exptr)
{
ExitState *ex = (ExitState *)exptr;
SnapNo snapno = J->exitno; /* For now, snapno == exitno. */
Trace *T = J->trace[J->parent];
SnapShot *snap = &T->snap[snapno];
BCReg s, nslots = snap->nslots;
IRRef2 *map = &T->snapmap[snap->mapofs];
IRRef2 *flinks = map + nslots + snap->nframelinks;
TValue *o, *newbase, *ntop;
BloomFilter rfilt = snap_renamefilter(T, snapno);
lua_State *L = J->L;
/* Make sure the stack is big enough for the slots from the snapshot. */
if (L->base + nslots >= L->maxstack) {
L->top = curr_topL(L);
lj_state_growstack(L, nslots - curr_proto(L)->framesize);
}
/* Fill stack slots with data from the registers and spill slots. */
newbase = NULL;
ntop = L->base;
for (s = 0, o = L->base-1; s < nslots; s++, o++) {
IRRef ref = snap_ref(map[s]);
if (ref) {
IRIns *ir = &T->ir[ref];
if (irref_isk(ref)) { /* Restore constant slot. */
lj_ir_kvalue(L, o, ir);
} else {
IRType1 t = ir->t;
RegSP rs = ir->prev;
if (LJ_UNLIKELY(bloomtest(rfilt, ref)))
rs = snap_renameref(T, snapno, ref, rs);
if (ra_hasspill(regsp_spill(rs))) { /* Restore from spill slot. */
int32_t *sps = &ex->spill[regsp_spill(rs)];
if (irt_isinteger(t)) {
setintV(o, *sps);
} else if (irt_isnum(t)) {
o->u64 = *(uint64_t *)sps;
} else {
lua_assert(!irt_ispri(t)); /* PRI refs never have a spill slot. */
setgcrefi(o->gcr, *sps);
setitype(o, irt_toitype(t));
}
} else if (ra_hasreg(regsp_reg(rs))) { /* Restore from register. */
Reg r = regsp_reg(rs);
if (irt_isinteger(t)) {
setintV(o, ex->gpr[r-RID_MIN_GPR]);
} else if (irt_isnum(t)) {
setnumV(o, ex->fpr[r-RID_MIN_FPR]);
} else {
if (!irt_ispri(t))
setgcrefi(o->gcr, ex->gpr[r-RID_MIN_GPR]);
setitype(o, irt_toitype(t));
}
} else { /* Restore frame slot. */
lua_assert(ir->o == IR_FRAME);
/* This works for both PTR and FUNC IR_FRAME. */
setgcrefp(o->fr.func, mref(T->ir[ir->op2].ptr, void));
if (s != 0) /* Do not overwrite link to previous frame. */
o->fr.tp.ftsz = (int32_t)*--flinks;
if (irt_isfunc(ir->t)) {
GCfunc *fn = gco2func(gcref(T->ir[ir->op2].gcr));
if (isluafunc(fn)) {
TValue *fs;
newbase = o+1;
fs = newbase + funcproto(fn)->framesize;
if (fs > ntop) ntop = fs; /* Update top for newly added frames. */
}
}
}
}
} else if (newbase) {
setnilV(o); /* Clear unreferenced slots of newly added frames. */
}
}
/* Helper for CAT. Coercion, iterative concat, __concat metamethod. */
TValue *lj_meta_cat(lua_State *L, TValue *top, int left)
{
int fromc = 0;
if (left < 0) { left = -left; fromc = 1; }
do {
if (!(tvisstr(top) || tvisnumber(top)) ||
!(tvisstr(top-1) || tvisnumber(top-1))) {
cTValue *mo = lj_meta_lookup(L, top-1, MM_concat);
if (tvisnil(mo)) {
mo = lj_meta_lookup(L, top, MM_concat);
if (tvisnil(mo)) {
if (tvisstr(top-1) || tvisnumber(top-1)) top++;
lj_err_optype(L, top-1, LJ_ERR_OPCAT);
return NULL; /* unreachable */
}
}
/* One of the top two elements is not a string, call __cat metamethod:
**
** before: [...][CAT stack .........................]
** top-1 top top+1 top+2
** pick two: [...][CAT stack ...] [o1] [o2]
** setup mm: [...][CAT stack ...] [cont|?] [mo|tmtype] [o1] [o2]
** in asm: [...][CAT stack ...] [cont|PC] [mo|delta] [o1] [o2]
** ^-- func base ^-- mm base
** after mm: [...][CAT stack ...] <--push-- [result]
** next step: [...][CAT stack .............]
*/
copyTV(L, top+2*LJ_FR2+2, top); /* Carefully ordered stack copies! */
copyTV(L, top+2*LJ_FR2+1, top-1);
copyTV(L, top+LJ_FR2, mo);
setcont(top-1, lj_cont_cat);
if (LJ_FR2) { setnilV(top); setnilV(top+2); top += 2; }
return top+1; /* Trigger metamethod call. */
} else {
/* Pick as many strings as possible from the top and concatenate them:
**
** before: [...][CAT stack ...........................]
** pick str: [...][CAT stack ...] [...... strings ......]
** concat: [...][CAT stack ...] [result]
** next step: [...][CAT stack ............]
*/
TValue *e, *o = top;
uint64_t tlen = tvisstr(o) ? strV(o)->len : STRFMT_MAXBUF_NUM;
char *p, *buf;
do {
o--; tlen += tvisstr(o) ? strV(o)->len : STRFMT_MAXBUF_NUM;
} while (--left > 0 && (tvisstr(o-1) || tvisnumber(o-1)));
if (tlen >= LJ_MAX_STR) lj_err_msg(L, LJ_ERR_STROV);
p = buf = lj_buf_tmp(L, (MSize)tlen);
for (e = top, top = o; o <= e; o++) {
if (tvisstr(o)) {
GCstr *s = strV(o);
MSize len = s->len;
p = lj_buf_wmem(p, strdata(s), len);
} else if (tvisint(o)) {
p = lj_strfmt_wint(p, intV(o));
} else {
lua_assert(tvisnum(o));
p = lj_strfmt_wnum(p, o);
}
}
setstrV(L, top, lj_str_new(L, buf, (size_t)(p-buf)));
}
} while (left >= 1);
if (LJ_UNLIKELY(G(L)->gc.total >= G(L)->gc.threshold)) {
if (!fromc) L->top = curr_topL(L);
lj_gc_step(L);
}
return NULL;
}
/* Restore interpreter state from exit state with the help of a snapshot. */
const BCIns *lj_snap_restore(jit_State *J, void *exptr)
{
ExitState *ex = (ExitState *)exptr;
SnapNo snapno = J->exitno; /* For now, snapno == exitno. */
GCtrace *T = traceref(J, J->parent);
SnapShot *snap = &T->snap[snapno];
MSize n, nent = snap->nent;
SnapEntry *map = &T->snapmap[snap->mapofs];
SnapEntry *flinks = map + nent + snap->depth;
int32_t ftsz0;
BCReg nslots = snap->nslots;
TValue *frame;
BloomFilter rfilt = snap_renamefilter(T, snapno);
const BCIns *pc = snap_pc(map[nent]);
lua_State *L = J->L;
/* Set interpreter PC to the next PC to get correct error messages. */
setcframe_pc(cframe_raw(L->cframe), pc+1);
/* Make sure the stack is big enough for the slots from the snapshot. */
if (LJ_UNLIKELY(L->base + nslots > tvref(L->maxstack))) {
L->top = curr_topL(L);
lj_state_growstack(L, nslots - curr_proto(L)->framesize);
}
/* Fill stack slots with data from the registers and spill slots. */
frame = L->base-1;
ftsz0 = frame_ftsz(frame); /* Preserve link to previous frame in slot #0. */
for (n = 0; n < nent; n++) {
SnapEntry sn = map[n];
IRRef ref = snap_ref(sn);
BCReg s = snap_slot(sn);
TValue *o = &frame[s]; /* Stack slots are relative to start frame. */
IRIns *ir = &T->ir[ref];
if (irref_isk(ref)) { /* Restore constant slot. */
lj_ir_kvalue(L, o, ir);
if ((sn & (SNAP_CONT|SNAP_FRAME))) {
/* Overwrite tag with frame link. */
o->fr.tp.ftsz = s != 0 ? (int32_t)*flinks-- : ftsz0;
if ((sn & SNAP_FRAME)) {
GCfunc *fn = ir_kfunc(ir);
if (isluafunc(fn)) {
MSize framesize = funcproto(fn)->framesize;
L->base = ++o;
if (LJ_UNLIKELY(o + framesize > tvref(L->maxstack))) {
ptrdiff_t fsave = savestack(L, frame);
L->top = o;
lj_state_growstack(L, framesize); /* Grow again. */
frame = restorestack(L, fsave);
}
}
}
}
} else if (!(sn & SNAP_NORESTORE)) {
IRType1 t = ir->t;
RegSP rs = ir->prev;
lua_assert(!(sn & (SNAP_CONT|SNAP_FRAME)));
if (LJ_UNLIKELY(bloomtest(rfilt, ref)))
rs = snap_renameref(T, snapno, ref, rs);
if (ra_hasspill(regsp_spill(rs))) { /* Restore from spill slot. */
int32_t *sps = &ex->spill[regsp_spill(rs)];
if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) {
o->u32.lo = (uint32_t)*sps;
} else if (irt_isinteger(t)) {
setintV(o, *sps);
#if !LJ_SOFTFP
} else if (irt_isnum(t)) {
o->u64 = *(uint64_t *)sps;
#endif
#if LJ_64
} else if (irt_islightud(t)) {
/* 64 bit lightuserdata which may escape already has the tag bits. */
o->u64 = *(uint64_t *)sps;
#endif
} else {
lua_assert(!irt_ispri(t)); /* PRI refs never have a spill slot. */
setgcrefi(o->gcr, *sps);
setitype(o, irt_toitype(t));
}
} else { /* Restore from register. */
Reg r = regsp_reg(rs);
lua_assert(ra_hasreg(r));
if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) {
o->u32.lo = (uint32_t)ex->gpr[r-RID_MIN_GPR];
} else if (irt_isinteger(t)) {
setintV(o, (int32_t)ex->gpr[r-RID_MIN_GPR]);
#if !LJ_SOFTFP
} else if (irt_isnum(t)) {
setnumV(o, ex->fpr[r-RID_MIN_FPR]);
#endif
#if LJ_64
} else if (irt_islightud(t)) {
/* 64 bit lightuserdata which may escape already has the tag bits. */
o->u64 = ex->gpr[r-RID_MIN_GPR];
#endif
} else {
if (!irt_ispri(t))
setgcrefi(o->gcr, ex->gpr[r-RID_MIN_GPR]);
setitype(o, irt_toitype(t));
}
}
if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) {
rs = (ir+1)->prev;
if (LJ_UNLIKELY(bloomtest(rfilt, ref+1)))
rs = snap_renameref(T, snapno, ref+1, rs);
o->u32.hi = (ra_hasspill(regsp_spill(rs))) ?
(uint32_t)*&ex->spill[regsp_spill(rs)] :
(uint32_t)ex->gpr[regsp_reg(rs)-RID_MIN_GPR];
}
}
}
switch (bc_op(*pc)) {
case BC_CALLM: case BC_CALLMT: case BC_RETM: case BC_TSETM:
L->top = frame + nslots;
break;
default:
L->top = curr_topL(L);
break;
}
lua_assert(map + nent == flinks);
return pc;
}
/* Restore interpreter state from exit state with the help of a snapshot. */
const BCIns *lj_snap_restore(jit_State *J, void *exptr)
{
ExitState *ex = (ExitState *)exptr;
SnapNo snapno = J->exitno; /* For now, snapno == exitno. */
GCtrace *T = traceref(J, J->parent);
SnapShot *snap = &T->snap[snapno];
MSize n, nent = snap->nent;
SnapEntry *map = &T->snapmap[snap->mapofs];
SnapEntry *flinks = &T->snapmap[snap_nextofs(T, snap)-1];
ptrdiff_t ftsz0;
TValue *frame;
BloomFilter rfilt = snap_renamefilter(T, snapno);
const BCIns *pc = snap_pc(map[nent]);
lua_State *L = J->L;
/* Set interpreter PC to the next PC to get correct error messages. */
setcframe_pc(cframe_raw(L->cframe), pc+1);
/* Make sure the stack is big enough for the slots from the snapshot. */
if (LJ_UNLIKELY(L->base + snap->topslot >= tvref(L->maxstack))) {
L->top = curr_topL(L);
lj_state_growstack(L, snap->topslot - curr_proto(L)->framesize);
}
/* Fill stack slots with data from the registers and spill slots. */
frame = L->base-1;
ftsz0 = frame_ftsz(frame); /* Preserve link to previous frame in slot #0. */
for (n = 0; n < nent; n++) {
SnapEntry sn = map[n];
if (!(sn & SNAP_NORESTORE)) {
TValue *o = &frame[snap_slot(sn)];
IRRef ref = snap_ref(sn);
IRIns *ir = &T->ir[ref];
if (ir->r == RID_SUNK) {
MSize j;
for (j = 0; j < n; j++)
if (snap_ref(map[j]) == ref) { /* De-duplicate sunk allocations. */
copyTV(L, o, &frame[snap_slot(map[j])]);
goto dupslot;
}
snap_unsink(J, T, ex, snapno, rfilt, ir, o);
dupslot:
continue;
}
snap_restoreval(J, T, ex, snapno, rfilt, ref, o);
if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM) && tvisint(o)) {
TValue tmp;
snap_restoreval(J, T, ex, snapno, rfilt, ref+1, &tmp);
o->u32.hi = tmp.u32.lo;
} else if ((sn & (SNAP_CONT|SNAP_FRAME))) {
lua_assert(!LJ_FR2); /* TODO_FR2: store 64 bit PCs. */
/* Overwrite tag with frame link. */
setframe_ftsz(o, snap_slot(sn) != 0 ? (int32_t)*flinks-- : ftsz0);
L->base = o+1;
}
}
}
lua_assert(map + nent == flinks);
/* Compute current stack top. */
switch (bc_op(*pc)) {
default:
if (bc_op(*pc) < BC_FUNCF) {
L->top = curr_topL(L);
break;
}
/* fallthrough */
case BC_CALLM: case BC_CALLMT: case BC_RETM: case BC_TSETM:
L->top = frame + snap->nslots;
break;
}
return pc;
}
