/* Find existing open upvalue for a stack slot or create a new one. */
static GCupval *func_finduv(lua_State *L, TValue *slot)
{
global_State *g = G(L);
GCRef *pp = &L->openupval;
GCupval *p;
GCupval *uv;
/* Search the sorted list of open upvalues. */
while (gcref(*pp) != NULL && uvval((p = gco2uv(gcref(*pp)))) >= slot) {
lua_assert(!p->closed && uvval(p) != &p->tv);
if (uvval(p) == slot) { /* Found open upvalue pointing to same slot? */
if (isdead(g, obj2gco(p))) /* Resurrect it, if it's dead. */
flipwhite(obj2gco(p));
return p;
}
pp = &p->nextgc;
}
/* No matching upvalue found. Create a new one. */
uv = lj_mem_newt(L, sizeof(GCupval), GCupval);
newwhite(g, uv);
uv->gct = ~LJ_TUPVAL;
uv->closed = 0; /* Still open. */
setmref(uv->v, slot); /* Pointing to the stack slot. */
/* NOBARRIER: The GCupval is new (marked white) and open. */
setgcrefr(uv->nextgc, *pp); /* Insert into sorted list of open upvalues. */
setgcref(*pp, obj2gco(uv));
setgcref(uv->prev, obj2gco(&g->uvhead)); /* Insert into GC list, too. */
setgcrefr(uv->next, g->uvhead.next);
setgcref(uvnext(uv)->prev, obj2gco(uv));
setgcref(g->uvhead.next, obj2gco(uv));
lua_assert(uvprev(uvnext(uv)) == uv && uvnext(uvprev(uv)) == uv);
return uv;
}
GCudata *lj_udata_new(lua_State *L, MSize sz, GCtab *env)
{
GCudata *ud = lj_mem_newt(L, sizeof(GCudata) + sz, GCudata);
global_State *g = G(L);
newwhite(g, ud); /* Not finalized. */
ud->gct = ~LJ_TUDATA;
ud->udtype = UDTYPE_USERDATA;
ud->len = sz;
/* NOBARRIER: The GCudata is new (marked white). */
setgcrefnull(ud->metatable);
setgcref(ud->env, obj2gco(env));
/* Chain to userdata list (after main thread). */
setgcrefr(ud->nextgc, mainthread(g)->nextgc);
setgcref(mainthread(g)->nextgc, obj2gco(ud));
return ud;
}
void lj_lib_prereg(lua_State *L, const char *name, lua_CFunction f, GCtab *env)
{
luaL_findtable(L, LUA_REGISTRYINDEX, "_PRELOAD", 4);
lua_pushcfunction(L, f);
/* NOBARRIER: The function is new (marked white). */
setgcref(funcV(L->top-1)->c.env, obj2gco(env));
lua_setfield(L, -2, name);
L->top--;
}
static GCfunc *func_newL(lua_State *L, GCproto *pt, GCtab *env)
{
GCfunc *fn = (GCfunc *)lj_mem_newgco(L, sizeLfunc((MSize)pt->sizeuv));
fn->l.gct = ~LJ_TFUNC;
fn->l.ffid = FF_LUA;
fn->l.nupvalues = 0; /* Set to zero until upvalues are initialized. */
/* NOBARRIER: Really a setgcref. But the GCfunc is new (marked white). */
setmref(fn->l.pc, proto_bc(pt));
setgcref(fn->l.env, obj2gco(env));
return fn;
}
GCfunc *lj_func_newC(lua_State *L, MSize nelems, GCtab *env)
{
GCfunc *fn = (GCfunc *)lj_mem_newgco(L, sizeCfunc(nelems));
fn->c.gct = ~LJ_TFUNC;
fn->c.ffid = FF_C;
fn->c.nupvalues = (uint8_t)nelems;
/* NOBARRIER: The GCfunc is new (marked white). */
setmref(fn->c.pc, &G(L)->bc_cfunc_ext);
setgcref(fn->c.env, obj2gco(env));
return fn;
}
/* Create a new CLibrary object and push it on the stack. */
static CLibrary *clib_new(lua_State *L, GCtab *mt)
{
GCtab *t = lj_tab_new(L, 0, 0);
GCudata *ud = lj_udata_new(L, sizeof(CLibrary), t);
CLibrary *cl = (CLibrary *)uddata(ud);
cl->cache = t;
ud->udtype = UDTYPE_FFI_CLIB;
/* NOBARRIER: The GCudata is new (marked white). */
setgcref(ud->metatable, obj2gco(mt));
setudataV(L, L->top++, ud);
return cl;
}
/* Create a new Lua function with empty upvalues. */
GCfunc *lj_func_newL_empty(lua_State *L, GCproto *pt, GCtab *env)
{
GCfunc *fn = func_newL(L, pt, env);
MSize i, nuv = pt->sizeuv;
/* NOBARRIER: The GCfunc is new (marked white). */
for (i = 0; i < nuv; i++) {
GCupval *uv = func_emptyuv(L);
uv->dhash = (uint32_t)(uintptr_t)pt ^ ((uint32_t)proto_uv(pt)[i] << 24);
setgcref(fn->l.uvptr[i], obj2gco(uv));
}
fn->l.nupvalues = (uint8_t)nuv;
return fn;
}
/* Free a C data object. */
void LJ_FASTCALL lj_cdata_free(global_State *g, GCcdata *cd)
{
if (LJ_UNLIKELY(cd->marked & LJ_GC_CDATA_FIN)) {
GCobj *root;
makewhite(g, obj2gco(cd));
markfinalized(obj2gco(cd));
if ((root = gcref(g->gc.mmudata)) != NULL) {
setgcrefr(cd->nextgc, root->gch.nextgc);
setgcref(root->gch.nextgc, obj2gco(cd));
setgcref(g->gc.mmudata, obj2gco(cd));
} else {
setgcref(cd->nextgc, obj2gco(cd));
setgcref(g->gc.mmudata, obj2gco(cd));
}
} else if (LJ_LIKELY(!cdataisv(cd))) {
CType *ct = ctype_raw(ctype_ctsG(g), cd->ctypeid);
CTSize sz = ctype_hassize(ct->info) ? ct->size : CTSIZE_PTR;
lua_assert(ctype_hassize(ct->info) || ctype_isfunc(ct->info) ||
ctype_isextern(ct->info));
lj_mem_free(g, cd, sizeof(GCcdata) + sz);
} else {
lj_mem_free(g, memcdatav(cd), sizecdatav(cd));
}
}
/* -- Luajittex needs this one because it overloads loadfile -- */
LUALIB_API int RESERVED_load_aux_JIT(lua_State *L, int status, int envarg)
{
if (status == 0) {
if (tvistab(L->base+envarg-1)) {
GCfunc *fn = funcV(L->top-1);
GCtab *t = tabV(L->base+envarg-1);
setgcref(fn->c.env, obj2gco(t));
lj_gc_objbarrier(L, fn, t);
}
return 1;
} else {
setnilV(L->top-2);
return 2;
}
}
static GCfunc *func_newL(lua_State *L, GCproto *pt, GCtab *env)
{
uint32_t count;
GCfunc *fn = (GCfunc *)lj_mem_newgco(L, sizeLfunc((MSize)pt->sizeuv));
fn->l.gct = ~LJ_TFUNC;
fn->l.ffid = FF_LUA;
fn->l.nupvalues = 0; /* Set to zero until upvalues are initialized. */
/* NOBARRIER: Really a setgcref. But the GCfunc is new (marked white). */
setmref(fn->l.pc, proto_bc(pt));
setgcref(fn->l.env, obj2gco(env));
/* Saturating 3 bit counter (0..7) for created closures. */
count = (uint32_t)pt->flags + PROTO_CLCOUNT;
pt->flags = (uint8_t)(count - ((count >> PROTO_CLC_BITS) & PROTO_CLCOUNT));
return fn;
}
/* String interning of metamethod names for fast indexing. */
void lj_meta_init(lua_State *L)
{
#define MMNAME(name) "__" #name
const char *metanames = MMDEF(MMNAME);
#undef MMNAME
global_State *g = G(L);
const char *p, *q;
uint32_t mm;
for (mm = 0, p = metanames; *p; mm++, p = q) {
GCstr *s;
for (q = p+2; *q && *q != '_'; q++) ;
s = lj_str_new(L, p, (size_t)(q-p));
/* NOBARRIER: g->gcroot[] is a GC root. */
setgcref(g->gcroot[GCROOT_MMNAME+mm], obj2gco(s));
}
}
/* Allocate variable-sized or specially aligned C data object. */
GCcdata *lj_cdata_newv(lua_State *L, CTypeID id, CTSize sz, CTSize align)
{
global_State *g;
MSize extra = sizeof(GCcdataVar) + sizeof(GCcdata) +
(align > CT_MEMALIGN ? (1u<<align) - (1u<<CT_MEMALIGN) : 0);
char *p = lj_mem_newt(L, extra + sz, char);
uintptr_t adata = (uintptr_t)p + sizeof(GCcdataVar) + sizeof(GCcdata);
uintptr_t almask = (1u << align) - 1u;
GCcdata *cd = (GCcdata *)(((adata + almask) & ~almask) - sizeof(GCcdata));
lua_assert((char *)cd - p < 65536);
cdatav(cd)->offset = (uint16_t)((char *)cd - p);
cdatav(cd)->extra = extra;
cdatav(cd)->len = sz;
g = G(L);
setgcrefr(cd->nextgc, g->gc.root);
setgcref(g->gc.root, obj2gco(cd));
newwhite(g, obj2gco(cd));
cd->marked |= 0x80;
cd->gct = ~LJ_TCDATA;
cd->ctypeid = id;
return cd;
}
/* Unsink allocation from the trace exit state. Unsink sunk stores. */
static void snap_unsink(jit_State *J, GCtrace *T, ExitState *ex,
SnapNo snapno, BloomFilter rfilt,
IRIns *ir, TValue *o)
{
lua_assert(ir->o == IR_TNEW || ir->o == IR_TDUP ||
ir->o == IR_CNEW || ir->o == IR_CNEWI);
#if LJ_HASFFI
if (ir->o == IR_CNEW || ir->o == IR_CNEWI) {
CTState *cts = ctype_cts(J->L);
CTypeID id = (CTypeID)T->ir[ir->op1].i;
CTSize sz;
CTInfo info = lj_ctype_info(cts, id, &sz);
GCcdata *cd = lj_cdata_newx(cts, id, sz, info);
setcdataV(J->L, o, cd);
if (ir->o == IR_CNEWI) {
uint8_t *p = (uint8_t *)cdataptr(cd);
lua_assert(sz == 4 || sz == 8);
if (LJ_32 && sz == 8 && ir+1 < T->ir + T->nins && (ir+1)->o == IR_HIOP) {
snap_restoredata(T, ex, snapno, rfilt, (ir+1)->op2, LJ_LE?p+4:p, 4);
if (LJ_BE) p += 4;
sz = 4;
}
snap_restoredata(T, ex, snapno, rfilt, ir->op2, p, sz);
} else {
IRIns *irs, *irlast = &T->ir[T->snap[snapno].ref];
for (irs = ir+1; irs < irlast; irs++)
if (irs->r == RID_SINK && snap_sunk_store(T, ir, irs)) {
IRIns *iro = &T->ir[T->ir[irs->op1].op2];
uint8_t *p = (uint8_t *)cd;
CTSize szs;
lua_assert(irs->o == IR_XSTORE && T->ir[irs->op1].o == IR_ADD);
lua_assert(iro->o == IR_KINT || iro->o == IR_KINT64);
if (irt_is64(irs->t)) szs = 8;
else if (irt_isi8(irs->t) || irt_isu8(irs->t)) szs = 1;
else if (irt_isi16(irs->t) || irt_isu16(irs->t)) szs = 2;
else szs = 4;
if (LJ_64 && iro->o == IR_KINT64)
p += (int64_t)ir_k64(iro)->u64;
else
p += iro->i;
lua_assert(p >= (uint8_t *)cdataptr(cd) &&
p + szs <= (uint8_t *)cdataptr(cd) + sz);
if (LJ_32 && irs+1 < T->ir + T->nins && (irs+1)->o == IR_HIOP) {
lua_assert(szs == 4);
snap_restoredata(T, ex, snapno, rfilt, (irs+1)->op2, LJ_LE?p+4:p,4);
if (LJ_BE) p += 4;
}
snap_restoredata(T, ex, snapno, rfilt, irs->op2, p, szs);
}
}
} else
#endif
{
IRIns *irs, *irlast;
GCtab *t = ir->o == IR_TNEW ? lj_tab_new(J->L, ir->op1, ir->op2) :
lj_tab_dup(J->L, ir_ktab(&T->ir[ir->op1]));
settabV(J->L, o, t);
irlast = &T->ir[T->snap[snapno].ref];
for (irs = ir+1; irs < irlast; irs++)
if (irs->r == RID_SINK && snap_sunk_store(T, ir, irs)) {
IRIns *irk = &T->ir[irs->op1];
TValue tmp, *val;
lua_assert(irs->o == IR_ASTORE || irs->o == IR_HSTORE ||
irs->o == IR_FSTORE);
if (irk->o == IR_FREF) {
lua_assert(irk->op2 == IRFL_TAB_META);
snap_restoreval(J, T, ex, snapno, rfilt, irs->op2, &tmp);
/* NOBARRIER: The table is new (marked white). */
setgcref(t->metatable, obj2gco(tabV(&tmp)));
} else {
irk = &T->ir[irk->op2];
if (irk->o == IR_KSLOT) irk = &T->ir[irk->op1];
lj_ir_kvalue(J->L, &tmp, irk);
val = lj_tab_set(J->L, t, &tmp);
/* NOBARRIER: The table is new (marked white). */
snap_restoreval(J, T, ex, snapno, rfilt, irs->op2, val);
if (LJ_SOFTFP && irs+1 < T->ir + T->nins && (irs+1)->o == IR_HIOP) {
snap_restoreval(J, T, ex, snapno, rfilt, (irs+1)->op2, &tmp);
val->u32.hi = tmp.u32.lo;
}
}
}
}
}
