static void buf_grow(SBuf *sb, MSize sz)
{
MSize osz = sbufsz(sb), len = sbuflen(sb), nsz = osz;
char *b;
if (nsz < LJ_MIN_SBUF) nsz = LJ_MIN_SBUF;
while (nsz < sz) nsz += nsz;
b = (char *)lj_mem_realloc(sbufL(sb), sbufB(sb), osz, nsz);
setmref(sb->b, b);
setmref(sb->p, b + len);
setmref(sb->e, b + nsz);
}
void LJ_FASTCALL lj_buf_shrink(lua_State *L, SBuf *sb)
{
char *b = sbufB(sb);
MSize osz = (MSize)(sbufE(sb) - b);
if (osz > 2*LJ_MIN_SBUF) {
MSize n = (MSize)(sbufP(sb) - b);
b = lj_mem_realloc(L, b, osz, (osz >> 1));
setmref(sb->b, b);
setmref(sb->p, b + n);
setmref(sb->e, b + (osz >> 1));
}
/* Create new hash part for table. */
static LJ_AINLINE void newhpart(lua_State *L, GCtab *t, uint32_t hbits)
{
uint32_t hsize;
Node *node;
lua_assert(hbits != 0);
if (hbits > LJ_MAX_HBITS)
lj_err_msg(L, LJ_ERR_TABOV);
hsize = 1u << hbits;
node = lj_mem_newvec(L, hsize, Node);
setmref(node->freetop, &node[hsize]);
setmref(t->node, node);
t->hmask = hsize-1;
}
/* Find existing open upvalue for a stack slot or create a new one. */
static GCupval *func_finduv(lua_State *L, TValue *slot)
{
global_State *g = G(L);
GCRef *pp = &L->openupval;
GCupval *p;
GCupval *uv;
/* Search the sorted list of open upvalues. */
while (gcref(*pp) != NULL && uvval((p = gco2uv(gcref(*pp)))) >= slot) {
lua_assert(!p->closed && uvval(p) != &p->tv);
if (uvval(p) == slot) { /* Found open upvalue pointing to same slot? */
if (isdead(g, obj2gco(p))) /* Resurrect it, if it's dead. */
flipwhite(obj2gco(p));
return p;
}
pp = &p->nextgc;
}
/* No matching upvalue found. Create a new one. */
uv = lj_mem_newt(L, sizeof(GCupval), GCupval);
newwhite(g, uv);
uv->gct = ~LJ_TUPVAL;
uv->closed = 0; /* Still open. */
setmref(uv->v, slot); /* Pointing to the stack slot. */
/* NOBARRIER: The GCupval is new (marked white) and open. */
setgcrefr(uv->nextgc, *pp); /* Insert into sorted list of open upvalues. */
setgcref(*pp, obj2gco(uv));
setgcref(uv->prev, obj2gco(&g->uvhead)); /* Insert into GC list, too. */
setgcrefr(uv->next, g->uvhead.next);
setgcref(uvnext(uv)->prev, obj2gco(uv));
setgcref(g->uvhead.next, obj2gco(uv));
lua_assert(uvprev(uvnext(uv)) == uv && uvnext(uvprev(uv)) == uv);
return uv;
}
/* Throw error. Find catch frame, unwind stack and continue. */
LJ_NOINLINE void LJ_FASTCALL lj_err_throw(lua_State *L, int errcode)
{
global_State *g = G(L);
lj_trace_abort(g);
g->saved_jit_base = g->jit_base;
setmref(g->jit_base, NULL);
L->status = 0;
#if LJ_UNWIND_EXT
err_raise_ext(errcode);
/*
** A return from this function signals a corrupt C stack that cannot be
** unwound. We have no choice but to call the panic function and exit.
**
** Usually this is caused by a C function without unwind information.
** This should never happen on x64, but may happen if you've manually
** enabled LUAJIT_UNWIND_EXTERNAL and forgot to recompile *every*
** non-C++ file with -funwind-tables.
*/
if (G(L)->panic)
G(L)->panic(L);
#else
{
void *cf = err_unwind(L, NULL, errcode);
if (cframe_unwind_ff(cf))
lj_vm_unwind_ff(cframe_raw(cf));
else
lj_vm_unwind_c(cframe_raw(cf), errcode);
}
#endif
exit(EXIT_FAILURE);
}
/* Stop profiling. */
LUA_API void luaJIT_profile_stop(lua_State *L)
{
ProfileState *ps = &profile_state;
global_State *g = ps->g;
if (G(L) == g) { /* Only stop profiler if started by this VM. */
profile_timer_stop(ps);
g->hookmask &= ~HOOK_PROFILE;
lj_dispatch_update(g);
#if LJ_HASJIT
G2J(g)->prof_mode = 0;
lj_trace_flushall(L);
#endif
lj_buf_free(g, &ps->sb);
setmref(ps->sb.b, NULL);
setmref(ps->sb.e, NULL);
ps->g = NULL;
}
}
/* Create an empty and closed upvalue. */
static GCupval *func_emptyuv(lua_State *L)
{
GCupval *uv = (GCupval *)lj_mem_newgco(L, sizeof(GCupval));
uv->gct = ~LJ_TUPVAL;
uv->closed = 1;
setnilV(&uv->tv);
setmref(uv->v, &uv->tv);
return uv;
}
/* Push internal function on the stack. */
GCfunc *lj_lib_pushcc(lua_State *L, lua_CFunction f, int id, int n)
{
GCfunc *fn;
lua_pushcclosure(L, f, n);
fn = funcV(L->top-1);
fn->c.ffid = (uint8_t)id;
setmref(fn->c.pc, &G(L)->bc_cfunc_int);
return fn;
}
static GCfunc *func_newL(lua_State *L, GCproto *pt, GCtab *env)
{
GCfunc *fn = (GCfunc *)lj_mem_newgco(L, sizeLfunc((MSize)pt->sizeuv));
fn->l.gct = ~LJ_TFUNC;
fn->l.ffid = FF_LUA;
fn->l.nupvalues = 0; /* Set to zero until upvalues are initialized. */
/* NOBARRIER: Really a setgcref. But the GCfunc is new (marked white). */
setmref(fn->l.pc, proto_bc(pt));
setgcref(fn->l.env, obj2gco(env));
return fn;
}
GCfunc *lj_func_newC(lua_State *L, MSize nelems, GCtab *env)
{
GCfunc *fn = (GCfunc *)lj_mem_newgco(L, sizeCfunc(nelems));
fn->c.gct = ~LJ_TFUNC;
fn->c.ffid = FF_C;
fn->c.nupvalues = (uint8_t)nelems;
/* NOBARRIER: The GCfunc is new (marked white). */
setmref(fn->c.pc, &G(L)->bc_cfunc_ext);
setgcref(fn->c.env, obj2gco(env));
return fn;
}
/* Clear hash part of table. */
static LJ_AINLINE void clearhpart(GCtab *t)
{
uint32_t i, hmask = t->hmask;
Node *node = noderef(t->node);
lua_assert(t->hmask != 0);
for (i = 0; i <= hmask; i++) {
Node *n = &node[i];
setmref(n->next, NULL);
setnilV(&n->key);
setnilV(&n->val);
}
}
/* Create a new table. Note: the slots are not initialized (yet). */
static GCtab *newtab(lua_State *L, uint32_t asize, uint32_t hbits)
{
GCtab *t;
/* First try to colocate the array part. */
if (LJ_MAX_COLOSIZE != 0 && asize > 0 && asize <= LJ_MAX_COLOSIZE) {
lua_assert((sizeof(GCtab) & 7) == 0);
t = (GCtab *)lj_mem_newgco(L, sizetabcolo(asize));
t->gct = ~LJ_TTAB;
t->nomm = (uint8_t)~0;
t->colo = (int8_t)asize;
setmref(t->array, (TValue *)((char *)t + sizeof(GCtab)));
setgcrefnull(t->metatable);
t->asize = asize;
t->hmask = 0;
setmref(t->node, &G(L)->nilnode);
}
else { /* Otherwise separately allocate the array part. */
t = lj_mem_newobj(L, GCtab);
t->gct = ~LJ_TTAB;
t->nomm = (uint8_t)~0;
t->colo = 0;
setmref(t->array, NULL);
setgcrefnull(t->metatable);
t->asize = 0; /* In case the array allocation fails. */
t->hmask = 0;
setmref(t->node, &G(L)->nilnode);
if (asize > 0) {
if (asize > LJ_MAX_ASIZE)
lj_err_msg(L, LJ_ERR_TABOV);
setmref(t->array, lj_mem_newvec(L, asize, TValue));
t->asize = asize;
}
}
if (hbits)
newhpart(L, t, hbits);
return t;
}
static GCfunc *func_newL(lua_State *L, GCproto *pt, GCtab *env)
{
uint32_t count;
GCfunc *fn = (GCfunc *)lj_mem_newgco(L, sizeLfunc((MSize)pt->sizeuv));
fn->l.gct = ~LJ_TFUNC;
fn->l.ffid = FF_LUA;
fn->l.nupvalues = 0; /* Set to zero until upvalues are initialized. */
/* NOBARRIER: Really a setgcref. But the GCfunc is new (marked white). */
setmref(fn->l.pc, proto_bc(pt));
setgcref(fn->l.env, obj2gco(env));
/* Saturating 3 bit counter (0..7) for created closures. */
count = (uint32_t)pt->flags + PROTO_CLCOUNT;
pt->flags = (uint8_t)(count - ((count >> PROTO_CLC_BITS) & PROTO_CLCOUNT));
return fn;
}
void lj_lib_register(lua_State *L, const char *libname,
const uint8_t *p, const lua_CFunction *cf)
{
GCtab *env = tabref(L->env);
GCfunc *ofn = NULL;
int ffid = *p++;
BCIns *bcff = &L2GG(L)->bcff[*p++];
GCtab *tab = lib_create_table(L, libname, *p++);
ptrdiff_t tpos = L->top - L->base;
/* Avoid barriers further down. */
lj_gc_anybarriert(L, tab);
tab->nomm = 0;
for (;;) {
uint32_t tag = *p++;
MSize len = tag & LIBINIT_LENMASK;
tag &= LIBINIT_TAGMASK;
if (tag != LIBINIT_STRING) {
const char *name;
MSize nuv = (MSize)(L->top - L->base - tpos);
GCfunc *fn = lj_func_newC(L, nuv, env);
if (nuv) {
L->top = L->base + tpos;
memcpy(fn->c.upvalue, L->top, sizeof(TValue)*nuv);
}
fn->c.ffid = (uint8_t)(ffid++);
name = (const char *)p;
p += len;
if (tag == LIBINIT_CF)
setmref(fn->c.pc, &G(L)->bc_cfunc_int);
else
setmref(fn->c.pc, bcff++);
if (tag == LIBINIT_ASM_)
fn->c.f = ofn->c.f; /* Copy handler from previous function. */
else
fn->c.f = *cf++; /* Get cf or handler from C function table. */
if (len) {
/* NOBARRIER: See above for common barrier. */
setfuncV(L, lj_tab_setstr(L, tab, lj_str_new(L, name, len)), fn);
}
ofn = fn;
} else {
switch (tag | len) {
case LIBINIT_SET:
L->top -= 2;
if (tvisstr(L->top+1) && strV(L->top+1)->len == 0)
env = tabV(L->top);
else /* NOBARRIER: See above for common barrier. */
copyTV(L, lj_tab_set(L, tab, L->top+1), L->top);
break;
case LIBINIT_NUMBER:
memcpy(&L->top->n, p, sizeof(double));
L->top++;
p += sizeof(double);
break;
case LIBINIT_COPY:
copyTV(L, L->top, L->top - *p++);
L->top++;
break;
case LIBINIT_LASTCL:
setfuncV(L, L->top++, ofn);
break;
case LIBINIT_FFID:
ffid++;
break;
case LIBINIT_END:
return;
default:
setstrV(L, L->top++, lj_str_new(L, (const char *)p, len));
p += len;
break;
}
}
}
}