/* Initialize instruction dispatch table and hot counters. */ void lj_dispatch_init(GG_State *GG) { uint32_t i; ASMFunction *disp = GG->dispatch; for (i = 0; i < GG_LEN_SDISP; i++) disp[GG_LEN_DDISP+i] = disp[i] = makeasmfunc(lj_bc_ofs[i]); for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++) disp[i] = makeasmfunc(lj_bc_ofs[i]); /* The JIT engine is off by default. luaopen_jit() turns it on. */ disp[BC_FORL] = disp[BC_IFORL]; disp[BC_ITERL] = disp[BC_IITERL]; disp[BC_LOOP] = disp[BC_ILOOP]; disp[BC_FUNCF] = disp[BC_IFUNCF]; disp[BC_FUNCV] = disp[BC_IFUNCV]; GG->g.bc_cfunc_ext = GG->g.bc_cfunc_int = BCINS_AD(BC_FUNCC, LUA_MINSTACK, 0); for (i = 0; i < GG_NUM_ASMFF; i++) GG->bcff[i] = BCINS_AD(BC__MAX+i, 0, 0); }
/* Update dispatch table depending on various flags. */ void lj_dispatch_update(global_State *g) { uint8_t oldmode = g->dispatchmode; uint8_t mode = 0; #if LJ_HASJIT mode |= (G2J(g)->flags & JIT_F_ON) ? 1 : 0; mode |= G2J(g)->state != LJ_TRACE_IDLE ? 6 : 0; #endif mode |= (g->hookmask & HOOK_EVENTMASK) ? 2 : 0; if (oldmode != mode) { /* Mode changed? */ ASMFunction *disp = G2GG(g)->dispatch; ASMFunction f_forl, f_iterl, f_loop; g->dispatchmode = mode; if ((mode & 5) == 1) { /* Hotcount if JIT is on, but not when recording. */ f_forl = makeasmfunc(lj_vm_op_ofs[BC_FORL]); f_iterl = makeasmfunc(lj_vm_op_ofs[BC_ITERL]); f_loop = makeasmfunc(lj_vm_op_ofs[BC_LOOP]); } else { /* Otherwise use the non-hotcounting instructions. */ f_forl = disp[GG_DISP_STATIC+BC_IFORL]; f_iterl = disp[GG_DISP_STATIC+BC_IITERL]; f_loop = disp[GG_DISP_STATIC+BC_ILOOP]; } /* Set static loop ins first (may be copied below). */ disp[GG_DISP_STATIC+BC_FORL] = f_forl; disp[GG_DISP_STATIC+BC_ITERL] = f_iterl; disp[GG_DISP_STATIC+BC_LOOP] = f_loop; if ((oldmode & 6) != (mode & 6)) { /* Need to change whole table? */ if ((mode & 6) == 0) { /* No hooks and no recording? */ /* Copy static dispatch table to dynamic dispatch table. */ memcpy(&disp[0], &disp[GG_DISP_STATIC], sizeof(ASMFunction)*BC__MAX); } else { /* The recording dispatch also checks for hooks. */ ASMFunction f = (mode & 6) == 6 ? lj_vm_record : lj_vm_hook; uint32_t i; for (i = 0; i < BC__MAX; i++) disp[i] = f; } } else if ((mode & 6) == 0) { /* Fix dynamic loop ins unless overriden. */ disp[BC_FORL] = f_forl; disp[BC_ITERL] = f_iterl; disp[BC_LOOP] = f_loop; } } }
/* Initialize instruction dispatch table and hot counters. */ void lj_dispatch_init(GG_State *GG) { uint32_t i; ASMFunction *disp = GG->dispatch; for (i = 0; i < BC__MAX; i++) disp[GG_DISP_STATIC+i] = disp[i] = makeasmfunc(lj_vm_op_ofs[i]); /* The JIT engine is off by default. luaopen_jit() turns it on. */ disp[BC_FORL] = disp[BC_IFORL]; disp[BC_ITERL] = disp[BC_IITERL]; disp[BC_LOOP] = disp[BC_ILOOP]; }
/* Call dispatch. Used by call hooks, hot calls or when recording. */ ASMFunction LJ_FASTCALL lj_dispatch_call(lua_State *L, const BCIns *pc) { GCfunc *fn = curr_func(L); BCOp op; global_State *g = G(L); #if LJ_HASJIT jit_State *J = G2J(g); #endif int missing = call_init(L, fn); #if LJ_HASJIT J->L = L; if ((uintptr_t)pc & 1) { /* Marker for hot call. */ pc = (const BCIns *)((uintptr_t)pc & ~(uintptr_t)1); lj_trace_hot(J, pc); goto out; } else if (J->state != LJ_TRACE_IDLE && !(g->hookmask & (HOOK_GC|HOOK_VMEVENT))) { /* Record the FUNC* bytecodes, too. */ lj_trace_ins(J, pc-1); /* The interpreter bytecode PC is offset by 1. */ } #endif if ((g->hookmask & LUA_MASKCALL)) { int i; for (i = 0; i < missing; i++) /* Add missing parameters. */ setnilV(L->top++); callhook(L, LUA_HOOKCALL, -1); /* Preserve modifications of missing parameters by lua_setlocal(). */ while (missing-- > 0 && tvisnil(L->top - 1)) L->top--; } #if LJ_HASJIT out: #endif op = bc_op(pc[-1]); /* Get FUNC* op. */ #if LJ_HASJIT /* Use the non-hotcounting variants if JIT is off or while recording. */ if ((!(J->flags & JIT_F_ON) || J->state != LJ_TRACE_IDLE) && (op == BC_FUNCF || op == BC_FUNCV)) op = (BCOp)((int)op+(int)BC_IFUNCF-(int)BC_FUNCF); #endif return makeasmfunc(lj_bc_ofs[op]); /* Return static dispatch target. */ }
/* Update dispatch table depending on various flags. */ void lj_dispatch_update(global_State *g) { uint8_t oldmode = g->dispatchmode; uint8_t mode = 0; #if LJ_HASJIT mode |= (G2J(g)->flags & JIT_F_ON) ? DISPMODE_JIT : 0; mode |= G2J(g)->state != LJ_TRACE_IDLE ? (DISPMODE_REC|DISPMODE_INS|DISPMODE_CALL) : 0; #endif #if LJ_HASPROFILE mode |= (g->hookmask & HOOK_PROFILE) ? (DISPMODE_PROF|DISPMODE_INS) : 0; #endif mode |= (g->hookmask & (LUA_MASKLINE|LUA_MASKCOUNT)) ? DISPMODE_INS : 0; mode |= (g->hookmask & LUA_MASKCALL) ? DISPMODE_CALL : 0; mode |= (g->hookmask & LUA_MASKRET) ? DISPMODE_RET : 0; if (oldmode != mode) { /* Mode changed? */ ASMFunction *disp = G2GG(g)->dispatch; ASMFunction f_forl, f_iterl, f_loop, f_funcf, f_funcv; g->dispatchmode = mode; /* Hotcount if JIT is on, but not while recording. */ if ((mode & (DISPMODE_JIT|DISPMODE_REC)) == DISPMODE_JIT) { f_forl = makeasmfunc(lj_bc_ofs[BC_FORL]); f_iterl = makeasmfunc(lj_bc_ofs[BC_ITERL]); f_loop = makeasmfunc(lj_bc_ofs[BC_LOOP]); f_funcf = makeasmfunc(lj_bc_ofs[BC_FUNCF]); f_funcv = makeasmfunc(lj_bc_ofs[BC_FUNCV]); } else { /* Otherwise use the non-hotcounting instructions. */ f_forl = disp[GG_LEN_DDISP+BC_IFORL]; f_iterl = disp[GG_LEN_DDISP+BC_IITERL]; f_loop = disp[GG_LEN_DDISP+BC_ILOOP]; f_funcf = makeasmfunc(lj_bc_ofs[BC_IFUNCF]); f_funcv = makeasmfunc(lj_bc_ofs[BC_IFUNCV]); } /* Init static counting instruction dispatch first (may be copied below). */ disp[GG_LEN_DDISP+BC_FORL] = f_forl; disp[GG_LEN_DDISP+BC_ITERL] = f_iterl; disp[GG_LEN_DDISP+BC_LOOP] = f_loop; /* Set dynamic instruction dispatch. */ if ((oldmode ^ mode) & (DISPMODE_PROF|DISPMODE_REC|DISPMODE_INS)) { /* Need to update the whole table. */ if (!(mode & DISPMODE_INS)) { /* No ins dispatch? */ /* Copy static dispatch table to dynamic dispatch table. */ memcpy(&disp[0], &disp[GG_LEN_DDISP], GG_LEN_SDISP*sizeof(ASMFunction)); /* Overwrite with dynamic return dispatch. */ if ((mode & DISPMODE_RET)) { disp[BC_RETM] = lj_vm_rethook; disp[BC_RET] = lj_vm_rethook; disp[BC_RET0] = lj_vm_rethook; disp[BC_RET1] = lj_vm_rethook; } } else { /* The recording dispatch also checks for hooks. */ ASMFunction f = (mode & DISPMODE_PROF) ? lj_vm_profhook : (mode & DISPMODE_REC) ? lj_vm_record : lj_vm_inshook; uint32_t i; for (i = 0; i < GG_LEN_SDISP; i++) disp[i] = f; } } else if (!(mode & DISPMODE_INS)) { /* Otherwise set dynamic counting ins. */ disp[BC_FORL] = f_forl; disp[BC_ITERL] = f_iterl; disp[BC_LOOP] = f_loop; /* Set dynamic return dispatch. */ if ((mode & DISPMODE_RET)) { disp[BC_RETM] = lj_vm_rethook; disp[BC_RET] = lj_vm_rethook; disp[BC_RET0] = lj_vm_rethook; disp[BC_RET1] = lj_vm_rethook; } else { disp[BC_RETM] = disp[GG_LEN_DDISP+BC_RETM]; disp[BC_RET] = disp[GG_LEN_DDISP+BC_RET]; disp[BC_RET0] = disp[GG_LEN_DDISP+BC_RET0]; disp[BC_RET1] = disp[GG_LEN_DDISP+BC_RET1]; } } /* Set dynamic call dispatch. */ if ((oldmode ^ mode) & DISPMODE_CALL) { /* Update the whole table? */ uint32_t i; if ((mode & DISPMODE_CALL) == 0) { /* No call hooks? */ for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++) disp[i] = makeasmfunc(lj_bc_ofs[i]); } else { for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++) disp[i] = lj_vm_callhook; } } if (!(mode & DISPMODE_CALL)) { /* Overwrite dynamic counting ins. */ disp[BC_FUNCF] = f_funcf; disp[BC_FUNCV] = f_funcv; } #if LJ_HASJIT /* Reset hotcounts for JIT off to on transition. */ if ((mode & DISPMODE_JIT) && !(oldmode & DISPMODE_JIT)) lj_dispatch_init_hotcount(g); #endif } }