/* Find existing open upvalue for a stack slot or create a new one. */
static GCupval *func_finduv(lua_State *L, TValue *slot)
{
global_State *g = G(L);
GCRef *pp = &L->openupval;
GCupval *p;
GCupval *uv;
/* Search the sorted list of open upvalues. */
while (gcref(*pp) != NULL && uvval((p = gco2uv(gcref(*pp)))) >= slot) {
lua_assert(!p->closed && uvval(p) != &p->tv);
if (uvval(p) == slot) { /* Found open upvalue pointing to same slot? */
if (isdead(g, obj2gco(p))) /* Resurrect it, if it's dead. */
flipwhite(obj2gco(p));
return p;
}
pp = &p->nextgc;
}
/* No matching upvalue found. Create a new one. */
uv = lj_mem_newt(L, sizeof(GCupval), GCupval);
newwhite(g, uv);
uv->gct = ~LJ_TUPVAL;
uv->closed = 0; /* Still open. */
setmref(uv->v, slot); /* Pointing to the stack slot. */
/* NOBARRIER: The GCupval is new (marked white) and open. */
setgcrefr(uv->nextgc, *pp); /* Insert into sorted list of open upvalues. */
setgcref(*pp, obj2gco(uv));
setgcref(uv->prev, obj2gco(&g->uvhead)); /* Insert into GC list, too. */
setgcrefr(uv->next, g->uvhead.next);
setgcref(uvnext(uv)->prev, obj2gco(uv));
setgcref(g->uvhead.next, obj2gco(uv));
lua_assert(uvprev(uvnext(uv)) == uv && uvnext(uvprev(uv)) == uv);
return uv;
}
/* Close all open upvalues pointing to some stack level or above. */
void LJ_FASTCALL lj_func_closeuv(lua_State *L, TValue *level)
{
GCupval *uv;
global_State *g = G(L);
while (gcref(L->openupval) != NULL &&
uvval((uv = gco2uv(gcref(L->openupval)))) >= level) {
GCobj *o = obj2gco(uv);
lua_assert(!isblack(o) && !uv->closed && uvval(uv) != &uv->tv);
setgcrefr(L->openupval, uv->nextgc); /* No longer in open list. */
if (isdead(g, o)) {
lj_func_freeuv(g, uv);
} else {
unlinkuv(uv);
lj_gc_closeuv(g, uv);
}
}
}
/* Get redirected or mangled external symbol. */
static const char *clib_extsym(CTState *cts, CType *ct, GCstr *name)
{
if (ct->sib) {
CType *ctf = ctype_get(cts, ct->sib);
if (ctype_isxattrib(ctf->info, CTA_REDIR))
return strdata(gco2str(gcref(ctf->name)));
}
return strdata(name);
}
/* Recursively set the JIT mode for all children of a prototype. */
static void setptmode_all(global_State *g, GCproto *pt, int mode)
{
ptrdiff_t i;
for (i = -(ptrdiff_t)pt->sizekgc; i < 0; i++) {
GCobj *o = gcref(pt->k.gc[i]);
if (o->gch.gct == ~LJ_TPROTO) {
setptmode(g, gco2pt(o), mode);
setptmode_all(g, gco2pt(o), mode);
}
}
}
NODE *lreadword()
{
NODE *val;
val = reader(readstream, "");
if (feof(readstream)) {
gcref(val);
return(NIL);
}
return(val);
}
NODE *lreadlist()
{
NODE *val;
val = parser(reader(readstream, ""), FALSE);
if (feof(readstream)) {
gcref(val);
return(Null_Word);
}
return(val);
}
NODE *cnv_node_to_numnode(NODE *ndi)
{
NODE *val;
int dr;
char s2[MAX_NUMBER], *s = s2;
if (is_number(ndi))
return (ndi);
ndi = cnv_node_to_strnode(ndi);
if (ndi == UNBOUND) return (UNBOUND);
if (((getstrlen(ndi)) < MAX_NUMBER) && (dr = numberp(ndi)))
{
if (backslashed(ndi))
noparity_strnzcpy(s, getstrptr(ndi), getstrlen(ndi));
else
strnzcpy(s, getstrptr(ndi), getstrlen(ndi));
if (*s == '+') ++s;
if (s2[getstrlen(ndi) - 1] == '.') s2[getstrlen(ndi) - 1] = 0;
if (/*TRUE || */ dr - 1 || getstrlen(ndi) > 9)
{
val = newnode(FLOAT);
setfloat(val, atof(s));
}
else
{
val = newnode(INT);
setint(val, atol(s));
}
gcref(ndi);
return (val);
}
else
{
gcref(ndi);
return (UNBOUND);
}
}
void lj_cdata_setfin(lua_State *L, GCcdata *cd, GCobj *obj, uint32_t it)
{
GCtab *t = ctype_ctsG(G(L))->finalizer;
if (gcref(t->metatable)) {
/* Add cdata to finalizer table, if still enabled. */
TValue *tv, tmp;
setcdataV(L, &tmp, cd);
lj_gc_anybarriert(L, t);
tv = lj_tab_set(L, t, &tmp);
setgcV(L, tv, obj, it);
if (!tvisnil(tv))
cd->marked |= LJ_GC_CDATA_FIN;
else
cd->marked &= ~LJ_GC_CDATA_FIN;
}
}
void silent_load(NODE *arg, char *prefix)
{
FILE *tmp_stream;
NODE *tmp_line, *exec_list;
char load_path[200];
NODE *st = valnode__caseobj(Startup);
int sv_val_status = val_status;
/* This procedure is called three ways:
* silent_load(NIL,*argv) loads *argv
* silent_load(proc,logolib) loads logolib/proc
* silent_load(proc,NULL) loads proc.lg
* The "/" or ".lg" is supplied by this procedure as needed.
*/
if (prefix == NULL && arg == NIL) return;
strcpy(load_path, (prefix == NULL ? "" : prefix));
if (arg != NIL) {
arg = cnv_node_to_strnode(arg);
if (arg == UNBOUND) return;
#ifdef unix
if (prefix != NULL) strcat(load_path, "/");
#endif
noparitylow_strnzcpy(load_path + (int)strlen(load_path),
getstrptr(arg), getstrlen(arg));
if (prefix == NULL) strcat(load_path, ".lg");
gcref(arg);
}
tmp_stream = loadstream;
tmp_line = vref(current_line);
loadstream = fopen(load_path, "r");
if (loadstream != NULL) {
while (!feof(loadstream) && NOT_THROWING) {
current_line = reref(current_line, reader(loadstream, ""));
exec_list =parser(current_line, TRUE);
val_status = 0;
if (exec_list != NIL) eval_driver(exec_list);
}
fclose(loadstream);
runstartup(st);
val_status = sv_val_status;
} else if (arg == NIL) ndprintf(stdout,"File not found: %t\n", prefix);
loadstream = tmp_stream;
deref(current_line);
current_line = tmp_line;
}
TValue * LJ_FASTCALL lj_cdata_setfin(lua_State *L, GCcdata *cd)
{
global_State *g = G(L);
GCtab *t = ctype_ctsG(g)->finalizer;
if (gcref(t->metatable)) {
/* Add cdata to finalizer table, if still enabled. */
TValue *tv, tmp;
setcdataV(L, &tmp, cd);
lj_gc_anybarriert(L, t);
tv = lj_tab_set(L, t, &tmp);
cd->marked |= LJ_GC_CDATA_FIN;
return tv;
} else {
/* Otherwise return dummy TValue. */
return &g->tmptv;
}
}
/* Flush a root trace. */
static void trace_flushroot(jit_State *J, GCtrace *T)
{
GCproto *pt = &gcref(T->startpt)->pt;
lua_assert(T->root == 0 && pt != NULL);
/* First unpatch any modified bytecode. */
trace_unpatch(J, T);
/* Unlink root trace from chain anchored in prototype. */
if (pt->trace == T->traceno) { /* Trace is first in chain. Easy. */
pt->trace = T->nextroot;
} else if (pt->trace) { /* Otherwise search in chain of root traces. */
GCtrace *T2 = traceref(J, pt->trace);
if (T2) {
for (; T2->nextroot; T2 = traceref(J, T2->nextroot))
if (T2->nextroot == T->traceno) {
T2->nextroot = T->nextroot; /* Unlink from chain. */
break;
}
}
}
}
static void perftools_addtrace(GCtrace *T)
{
static FILE *fp;
GCproto *pt = &gcref(T->startpt)->pt;
const BCIns *startpc = mref(T->startpc, const BCIns);
const char *name = proto_chunknamestr(pt);
BCLine lineno;
if (name[0] == '@' || name[0] == '=')
name++;
else
name = "(string)";
lua_assert(startpc >= proto_bc(pt) && startpc < proto_bc(pt) + pt->sizebc);
lineno = lj_debug_line(pt, proto_bcpos(pt, startpc));
if (!fp) {
char fname[40];
sprintf(fname, "/tmp/perf-%d.map", getpid());
if (!(fp = fopen(fname, "w"))) return;
setlinebuf(fp);
}
fprintf(fp, "%lx %x TRACE_%d::%s:%u\n",
(long)T->mcode, T->szmcode, T->traceno, name, lineno);
}
/* Free a C data object. */
void LJ_FASTCALL lj_cdata_free(global_State *g, GCcdata *cd)
{
if (LJ_UNLIKELY(cd->marked & LJ_GC_CDATA_FIN)) {
GCobj *root;
makewhite(g, obj2gco(cd));
markfinalized(obj2gco(cd));
if ((root = gcref(g->gc.mmudata)) != NULL) {
setgcrefr(cd->nextgc, root->gch.nextgc);
setgcref(root->gch.nextgc, obj2gco(cd));
setgcref(g->gc.mmudata, obj2gco(cd));
} else {
setgcref(cd->nextgc, obj2gco(cd));
setgcref(g->gc.mmudata, obj2gco(cd));
}
} else if (LJ_LIKELY(!cdataisv(cd))) {
CType *ct = ctype_raw(ctype_ctsG(g), cd->ctypeid);
CTSize sz = ctype_hassize(ct->info) ? ct->size : CTSIZE_PTR;
lua_assert(ctype_hassize(ct->info) || ctype_isfunc(ct->info) ||
ctype_isextern(ct->info));
lj_mem_free(g, cd, sizeof(GCcdata) + sz);
} else {
lj_mem_free(g, memcdatav(cd), sizecdatav(cd));
}
}
/* Find unused slots with reaching-definitions bytecode data-flow analysis. */
static BCReg snap_usedef(jit_State *J, uint8_t *udf,
const BCIns *pc, BCReg maxslot)
{
BCReg s;
GCobj *o;
if (maxslot == 0) return 0;
#ifdef LUAJIT_USE_VALGRIND
/* Avoid errors for harmless reads beyond maxslot. */
memset(udf, 1, SNAP_USEDEF_SLOTS);
#else
memset(udf, 1, maxslot);
#endif
/* Treat open upvalues as used. */
o = gcref(J->L->openupval);
while (o) {
if (uvval(gco2uv(o)) < J->L->base) break;
udf[uvval(gco2uv(o)) - J->L->base] = 0;
o = gcref(o->gch.nextgc);
}
#define USE_SLOT(s) udf[(s)] &= ~1
#define DEF_SLOT(s) udf[(s)] *= 3
/* Scan through following bytecode and check for uses/defs. */
lua_assert(pc >= proto_bc(J->pt) && pc < proto_bc(J->pt) + J->pt->sizebc);
for (;;) {
BCIns ins = *pc++;
BCOp op = bc_op(ins);
switch (bcmode_b(op)) {
case BCMvar: USE_SLOT(bc_b(ins)); break;
default: break;
}
switch (bcmode_c(op)) {
case BCMvar: USE_SLOT(bc_c(ins)); break;
case BCMrbase:
lua_assert(op == BC_CAT);
for (s = bc_b(ins); s <= bc_c(ins); s++) USE_SLOT(s);
for (; s < maxslot; s++) DEF_SLOT(s);
break;
case BCMjump:
handle_jump: {
BCReg minslot = bc_a(ins);
if (op >= BC_FORI && op <= BC_JFORL) minslot += FORL_EXT;
else if (op >= BC_ITERL && op <= BC_JITERL) minslot += bc_b(pc[-2])-1;
else if (op == BC_UCLO) { pc += bc_j(ins); break; }
for (s = minslot; s < maxslot; s++) DEF_SLOT(s);
return minslot < maxslot ? minslot : maxslot;
}
case BCMlit:
if (op == BC_JFORL || op == BC_JITERL || op == BC_JLOOP) {
goto handle_jump;
} else if (bc_isret(op)) {
BCReg top = op == BC_RETM ? maxslot : (bc_a(ins) + bc_d(ins)-1);
for (s = 0; s < bc_a(ins); s++) DEF_SLOT(s);
for (; s < top; s++) USE_SLOT(s);
for (; s < maxslot; s++) DEF_SLOT(s);
return 0;
}
break;
case BCMfunc: return maxslot; /* NYI: will abort, anyway. */
default: break;
}
switch (bcmode_a(op)) {
case BCMvar: USE_SLOT(bc_a(ins)); break;
case BCMdst:
if (!(op == BC_ISTC || op == BC_ISFC)) DEF_SLOT(bc_a(ins));
break;
case BCMbase:
if (op >= BC_CALLM && op <= BC_VARG) {
BCReg top = (op == BC_CALLM || op == BC_CALLMT || bc_c(ins) == 0) ?
maxslot : (bc_a(ins) + bc_c(ins)+LJ_FR2);
if (LJ_FR2) DEF_SLOT(bc_a(ins)+1);
s = bc_a(ins) - ((op == BC_ITERC || op == BC_ITERN) ? 3 : 0);
for (; s < top; s++) USE_SLOT(s);
for (; s < maxslot; s++) DEF_SLOT(s);
if (op == BC_CALLT || op == BC_CALLMT) {
for (s = 0; s < bc_a(ins); s++) DEF_SLOT(s);
return 0;
}
} else if (op == BC_KNIL) {
for (s = bc_a(ins); s <= bc_d(ins); s++) DEF_SLOT(s);
} else if (op == BC_TSETM) {
for (s = bc_a(ins)-1; s < maxslot; s++) USE_SLOT(s);
}
break;
default: break;
}
lua_assert(pc >= proto_bc(J->pt) && pc < proto_bc(J->pt) + J->pt->sizebc);
}
#undef USE_SLOT
#undef DEF_SLOT
return 0; /* unreachable */
}
MSize i, nuv;
TValue *base;
lj_gc_check_fixtop(L);
fn = func_newL(L, pt, tabref(parent->env));
/* NOBARRIER: The GCfunc is new (marked white). */
puv = parent->uvptr;
nuv = pt->sizeuv;
base = L->base;
for (i = 0; i < nuv; i++) {
uint32_t v = proto_uv(pt)[i];
GCupval *uv;
if ((v & 0x8000)) {
uv = func_finduv(L, base + (v & 0xff));
uv->dhash = (uint32_t)(uintptr_t)mref(parent->pc, char) ^ (v << 24);
} else {
uv = &gcref(puv[v])->uv;
}
setgcref(fn->l.uvptr[i], obj2gco(uv));
}
fn->l.nupvalues = (uint8_t)nuv;
return fn;
}
void LJ_FASTCALL lj_func_free(global_State *g, GCfunc *fn)
{
MSize size = isluafunc(fn) ? sizeLfunc((MSize)fn->l.nupvalues) :
sizeCfunc((MSize)fn->c.nupvalues);
lj_mem_free(g, fn, size);
}
/* Restore interpreter state from exit state with the help of a snapshot. */
void lj_snap_restore(jit_State *J, void *exptr)
{
ExitState *ex = (ExitState *)exptr;
SnapNo snapno = J->exitno; /* For now, snapno == exitno. */
Trace *T = J->trace[J->parent];
SnapShot *snap = &T->snap[snapno];
BCReg s, nslots = snap->nslots;
IRRef2 *map = &T->snapmap[snap->mapofs];
IRRef2 *flinks = map + nslots + snap->nframelinks;
TValue *o, *newbase, *ntop;
BloomFilter rfilt = snap_renamefilter(T, snapno);
lua_State *L = J->L;
/* Make sure the stack is big enough for the slots from the snapshot. */
if (L->base + nslots >= L->maxstack) {
L->top = curr_topL(L);
lj_state_growstack(L, nslots - curr_proto(L)->framesize);
}
/* Fill stack slots with data from the registers and spill slots. */
newbase = NULL;
ntop = L->base;
for (s = 0, o = L->base-1; s < nslots; s++, o++) {
IRRef ref = snap_ref(map[s]);
if (ref) {
IRIns *ir = &T->ir[ref];
if (irref_isk(ref)) { /* Restore constant slot. */
lj_ir_kvalue(L, o, ir);
} else {
IRType1 t = ir->t;
RegSP rs = ir->prev;
if (LJ_UNLIKELY(bloomtest(rfilt, ref)))
rs = snap_renameref(T, snapno, ref, rs);
if (ra_hasspill(regsp_spill(rs))) { /* Restore from spill slot. */
int32_t *sps = &ex->spill[regsp_spill(rs)];
if (irt_isinteger(t)) {
setintV(o, *sps);
} else if (irt_isnum(t)) {
o->u64 = *(uint64_t *)sps;
} else {
lua_assert(!irt_ispri(t)); /* PRI refs never have a spill slot. */
setgcrefi(o->gcr, *sps);
setitype(o, irt_toitype(t));
}
} else if (ra_hasreg(regsp_reg(rs))) { /* Restore from register. */
Reg r = regsp_reg(rs);
if (irt_isinteger(t)) {
setintV(o, ex->gpr[r-RID_MIN_GPR]);
} else if (irt_isnum(t)) {
setnumV(o, ex->fpr[r-RID_MIN_FPR]);
} else {
if (!irt_ispri(t))
setgcrefi(o->gcr, ex->gpr[r-RID_MIN_GPR]);
setitype(o, irt_toitype(t));
}
} else { /* Restore frame slot. */
lua_assert(ir->o == IR_FRAME);
/* This works for both PTR and FUNC IR_FRAME. */
setgcrefp(o->fr.func, mref(T->ir[ir->op2].ptr, void));
if (s != 0) /* Do not overwrite link to previous frame. */
o->fr.tp.ftsz = (int32_t)*--flinks;
if (irt_isfunc(ir->t)) {
GCfunc *fn = gco2func(gcref(T->ir[ir->op2].gcr));
if (isluafunc(fn)) {
TValue *fs;
newbase = o+1;
fs = newbase + funcproto(fn)->framesize;
if (fs > ntop) ntop = fs; /* Update top for newly added frames. */
}
}
}
}
} else if (newbase) {
setnilV(o); /* Clear unreferenced slots of newly added frames. */
}
}
