/* Initialize hotcount table. */
void lj_dispatch_init_hotcount(global_State *g)
{
HotCount start = (HotCount)G2J(g)->param[JIT_P_hotloop];
HotCount *hotcount = G2GG(g)->hotcount;
uint32_t i;
for (i = 0; i < HOTCOUNT_SIZE; i++)
hotcount[i] = start;
}
/* Update dispatch table depending on various flags. */
void lj_dispatch_update(global_State *g)
{
uint8_t oldmode = g->dispatchmode;
uint8_t mode = 0;
#if LJ_HASJIT
mode |= (G2J(g)->flags & JIT_F_ON) ? 1 : 0;
mode |= G2J(g)->state != LJ_TRACE_IDLE ? 6 : 0;
#endif
mode |= (g->hookmask & HOOK_EVENTMASK) ? 2 : 0;
if (oldmode != mode) { /* Mode changed? */
ASMFunction *disp = G2GG(g)->dispatch;
ASMFunction f_forl, f_iterl, f_loop;
g->dispatchmode = mode;
if ((mode & 5) == 1) { /* Hotcount if JIT is on, but not when recording. */
f_forl = makeasmfunc(lj_vm_op_ofs[BC_FORL]);
f_iterl = makeasmfunc(lj_vm_op_ofs[BC_ITERL]);
f_loop = makeasmfunc(lj_vm_op_ofs[BC_LOOP]);
} else { /* Otherwise use the non-hotcounting instructions. */
f_forl = disp[GG_DISP_STATIC+BC_IFORL];
f_iterl = disp[GG_DISP_STATIC+BC_IITERL];
f_loop = disp[GG_DISP_STATIC+BC_ILOOP];
}
/* Set static loop ins first (may be copied below). */
disp[GG_DISP_STATIC+BC_FORL] = f_forl;
disp[GG_DISP_STATIC+BC_ITERL] = f_iterl;
disp[GG_DISP_STATIC+BC_LOOP] = f_loop;
if ((oldmode & 6) != (mode & 6)) { /* Need to change whole table? */
if ((mode & 6) == 0) { /* No hooks and no recording? */
/* Copy static dispatch table to dynamic dispatch table. */
memcpy(&disp[0], &disp[GG_DISP_STATIC], sizeof(ASMFunction)*BC__MAX);
} else {
/* The recording dispatch also checks for hooks. */
ASMFunction f = (mode & 6) == 6 ? lj_vm_record : lj_vm_hook;
uint32_t i;
for (i = 0; i < BC__MAX; i++)
disp[i] = f;
}
} else if ((mode & 6) == 0) { /* Fix dynamic loop ins unless overriden. */
disp[BC_FORL] = f_forl;
disp[BC_ITERL] = f_iterl;
disp[BC_LOOP] = f_loop;
}
}
}
/* Update dispatch table depending on various flags. */
void lj_dispatch_update(global_State *g)
{
uint8_t oldmode = g->dispatchmode;
uint8_t mode = 0;
#if LJ_HASJIT
mode |= (G2J(g)->flags & JIT_F_ON) ? DISPMODE_JIT : 0;
mode |= G2J(g)->state != LJ_TRACE_IDLE ?
(DISPMODE_REC|DISPMODE_INS|DISPMODE_CALL) : 0;
#endif
#if LJ_HASPROFILE
mode |= (g->hookmask & HOOK_PROFILE) ? (DISPMODE_PROF|DISPMODE_INS) : 0;
#endif
mode |= (g->hookmask & (LUA_MASKLINE|LUA_MASKCOUNT)) ? DISPMODE_INS : 0;
mode |= (g->hookmask & LUA_MASKCALL) ? DISPMODE_CALL : 0;
mode |= (g->hookmask & LUA_MASKRET) ? DISPMODE_RET : 0;
if (oldmode != mode) { /* Mode changed? */
ASMFunction *disp = G2GG(g)->dispatch;
ASMFunction f_forl, f_iterl, f_loop, f_funcf, f_funcv;
g->dispatchmode = mode;
/* Hotcount if JIT is on, but not while recording. */
if ((mode & (DISPMODE_JIT|DISPMODE_REC)) == DISPMODE_JIT) {
f_forl = makeasmfunc(lj_bc_ofs[BC_FORL]);
f_iterl = makeasmfunc(lj_bc_ofs[BC_ITERL]);
f_loop = makeasmfunc(lj_bc_ofs[BC_LOOP]);
f_funcf = makeasmfunc(lj_bc_ofs[BC_FUNCF]);
f_funcv = makeasmfunc(lj_bc_ofs[BC_FUNCV]);
} else { /* Otherwise use the non-hotcounting instructions. */
f_forl = disp[GG_LEN_DDISP+BC_IFORL];
f_iterl = disp[GG_LEN_DDISP+BC_IITERL];
f_loop = disp[GG_LEN_DDISP+BC_ILOOP];
f_funcf = makeasmfunc(lj_bc_ofs[BC_IFUNCF]);
f_funcv = makeasmfunc(lj_bc_ofs[BC_IFUNCV]);
}
/* Init static counting instruction dispatch first (may be copied below). */
disp[GG_LEN_DDISP+BC_FORL] = f_forl;
disp[GG_LEN_DDISP+BC_ITERL] = f_iterl;
disp[GG_LEN_DDISP+BC_LOOP] = f_loop;
/* Set dynamic instruction dispatch. */
if ((oldmode ^ mode) & (DISPMODE_PROF|DISPMODE_REC|DISPMODE_INS)) {
/* Need to update the whole table. */
if (!(mode & DISPMODE_INS)) { /* No ins dispatch? */
/* Copy static dispatch table to dynamic dispatch table. */
memcpy(&disp[0], &disp[GG_LEN_DDISP], GG_LEN_SDISP*sizeof(ASMFunction));
/* Overwrite with dynamic return dispatch. */
if ((mode & DISPMODE_RET)) {
disp[BC_RETM] = lj_vm_rethook;
disp[BC_RET] = lj_vm_rethook;
disp[BC_RET0] = lj_vm_rethook;
disp[BC_RET1] = lj_vm_rethook;
}
} else {
/* The recording dispatch also checks for hooks. */
ASMFunction f = (mode & DISPMODE_PROF) ? lj_vm_profhook :
(mode & DISPMODE_REC) ? lj_vm_record : lj_vm_inshook;
uint32_t i;
for (i = 0; i < GG_LEN_SDISP; i++)
disp[i] = f;
}
} else if (!(mode & DISPMODE_INS)) {
/* Otherwise set dynamic counting ins. */
disp[BC_FORL] = f_forl;
disp[BC_ITERL] = f_iterl;
disp[BC_LOOP] = f_loop;
/* Set dynamic return dispatch. */
if ((mode & DISPMODE_RET)) {
disp[BC_RETM] = lj_vm_rethook;
disp[BC_RET] = lj_vm_rethook;
disp[BC_RET0] = lj_vm_rethook;
disp[BC_RET1] = lj_vm_rethook;
} else {
disp[BC_RETM] = disp[GG_LEN_DDISP+BC_RETM];
disp[BC_RET] = disp[GG_LEN_DDISP+BC_RET];
disp[BC_RET0] = disp[GG_LEN_DDISP+BC_RET0];
disp[BC_RET1] = disp[GG_LEN_DDISP+BC_RET1];
}
}
/* Set dynamic call dispatch. */
if ((oldmode ^ mode) & DISPMODE_CALL) { /* Update the whole table? */
uint32_t i;
if ((mode & DISPMODE_CALL) == 0) { /* No call hooks? */
for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)
disp[i] = makeasmfunc(lj_bc_ofs[i]);
} else {
for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)
disp[i] = lj_vm_callhook;
}
}
if (!(mode & DISPMODE_CALL)) { /* Overwrite dynamic counting ins. */
disp[BC_FUNCF] = f_funcf;
disp[BC_FUNCV] = f_funcv;
}
#if LJ_HASJIT
/* Reset hotcounts for JIT off to on transition. */
if ((mode & DISPMODE_JIT) && !(oldmode & DISPMODE_JIT))
lj_dispatch_init_hotcount(g);
#endif
}
}