/* Flush all traces. */ int lj_trace_flushall(lua_State *L) { jit_State *J = L2J(L); ptrdiff_t i; if ((J2G(J)->hookmask & HOOK_GC)) return 1; for (i = (ptrdiff_t)J->sizetrace-1; i > 0; i--) { GCtrace *T = traceref(J, i); if (T) { if (T->root == 0) trace_flushroot(J, T); lj_gdbjit_deltrace(J, T); T->traceno = 0; setgcrefnull(J->trace[i]); } } J->cur.traceno = 0; J->freetrace = 0; /* Clear penalty cache. */ memset(J->penalty, 0, sizeof(J->penalty)); /* Free the whole machine code and invalidate all exit stub groups. */ lj_mcode_free(J); memset(J->exitstubgroup, 0, sizeof(J->exitstubgroup)); lj_vmevent_send(L, TRACE, setstrV(L, L->top++, lj_str_newlit(L, "flush")); );
/* Check trace argument. Must not throw for non-existent trace numbers. */ static GCtrace *jit_checktrace(lua_State *L) { TraceNo tr = (TraceNo)lj_lib_checkint(L, 1); jit_State *J = L2J(L); if (tr > 0 && tr < J->sizetrace) return traceref(J, tr); return NULL; }
/* Flush a trace. Only root traces are considered. */ void lj_trace_flush(jit_State *J, TraceNo traceno) { if (traceno > 0 && traceno < J->sizetrace) { GCtrace *T = traceref(J, traceno); if (T && T->root == 0) trace_flushroot(J, T); } }
/* Flush a root trace. */ static void trace_flushroot(jit_State *J, GCtrace *T) { GCproto *pt = &gcref(T->startpt)->pt; lua_assert(T->root == 0 && pt != NULL); /* First unpatch any modified bytecode. */ trace_unpatch(J, T); /* Unlink root trace from chain anchored in prototype. */ if (pt->trace == T->traceno) { /* Trace is first in chain. Easy. */ pt->trace = T->nextroot; } else if (pt->trace) { /* Otherwise search in chain of root traces. */ GCtrace *T2 = traceref(J, pt->trace); if (T2) { for (; T2->nextroot; T2 = traceref(J, T2->nextroot)) if (T2->nextroot == T->traceno) { T2->nextroot = T->nextroot; /* Unlink from chain. */ break; } } } }
/* Unpatch the bytecode modified by a root trace. */ static void trace_unpatch(jit_State *J, GCtrace *T) { BCOp op = bc_op(T->startins); BCIns *pc = mref(T->startpc, BCIns); UNUSED(J); if (op == BC_JMP) return; /* No need to unpatch branches in parent traces (yet). */ switch (bc_op(*pc)) { case BC_JFORL: lua_assert(traceref(J, bc_d(*pc)) == T); *pc = T->startins; pc += bc_j(T->startins); lua_assert(bc_op(*pc) == BC_JFORI); setbc_op(pc, BC_FORI); break; case BC_JITERL: case BC_JLOOP: lua_assert(op == BC_ITERL || op == BC_LOOP || bc_isret(op)); *pc = T->startins; break; case BC_JMP: lua_assert(op == BC_ITERL); pc += bc_j(*pc)+2; if (bc_op(*pc) == BC_JITERL) { lua_assert(traceref(J, bc_d(*pc)) == T); *pc = T->startins; } break; case BC_JFUNCF: lua_assert(op == BC_FUNCF); *pc = T->startins; break; default: /* Already unpatched. */ break; } }
/* Find a free trace number. */ static TraceNo trace_findfree(jit_State *J) { MSize osz, lim; if (J->freetrace == 0) J->freetrace = 1; for (; J->freetrace < J->sizetrace; J->freetrace++) if (traceref(J, J->freetrace) == NULL) return J->freetrace++; /* Need to grow trace array. */ lim = (MSize)J->param[JIT_P_maxtrace] + 1; if (lim < 2) lim = 2; else if (lim > 65535) lim = 65535; osz = J->sizetrace; if (osz >= lim) return 0; /* Too many traces. */ lj_mem_growvec(J->L, J->trace, J->sizetrace, lim, GCRef); for (; osz < J->sizetrace; osz++) setgcrefnull(J->trace[osz]); return J->freetrace; }
/* Flush all traces associated with a prototype. */ void lj_trace_flushproto(global_State *g, GCproto *pt) { while (pt->trace != 0) trace_flushroot(G2J(g), traceref(G2J(g), pt->trace)); }
/* Restore interpreter state from exit state with the help of a snapshot. */ const BCIns *lj_snap_restore(jit_State *J, void *exptr) { ExitState *ex = (ExitState *)exptr; SnapNo snapno = J->exitno; /* For now, snapno == exitno. */ GCtrace *T = traceref(J, J->parent); SnapShot *snap = &T->snap[snapno]; MSize n, nent = snap->nent; SnapEntry *map = &T->snapmap[snap->mapofs]; SnapEntry *flinks = &T->snapmap[snap_nextofs(T, snap)-1]; ptrdiff_t ftsz0; TValue *frame; BloomFilter rfilt = snap_renamefilter(T, snapno); const BCIns *pc = snap_pc(map[nent]); lua_State *L = J->L; /* Set interpreter PC to the next PC to get correct error messages. */ setcframe_pc(cframe_raw(L->cframe), pc+1); /* Make sure the stack is big enough for the slots from the snapshot. */ if (LJ_UNLIKELY(L->base + snap->topslot >= tvref(L->maxstack))) { L->top = curr_topL(L); lj_state_growstack(L, snap->topslot - curr_proto(L)->framesize); } /* Fill stack slots with data from the registers and spill slots. */ frame = L->base-1; ftsz0 = frame_ftsz(frame); /* Preserve link to previous frame in slot #0. */ for (n = 0; n < nent; n++) { SnapEntry sn = map[n]; if (!(sn & SNAP_NORESTORE)) { TValue *o = &frame[snap_slot(sn)]; IRRef ref = snap_ref(sn); IRIns *ir = &T->ir[ref]; if (ir->r == RID_SUNK) { MSize j; for (j = 0; j < n; j++) if (snap_ref(map[j]) == ref) { /* De-duplicate sunk allocations. */ copyTV(L, o, &frame[snap_slot(map[j])]); goto dupslot; } snap_unsink(J, T, ex, snapno, rfilt, ir, o); dupslot: continue; } snap_restoreval(J, T, ex, snapno, rfilt, ref, o); if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM) && tvisint(o)) { TValue tmp; snap_restoreval(J, T, ex, snapno, rfilt, ref+1, &tmp); o->u32.hi = tmp.u32.lo; } else if ((sn & (SNAP_CONT|SNAP_FRAME))) { lua_assert(!LJ_FR2); /* TODO_FR2: store 64 bit PCs. */ /* Overwrite tag with frame link. */ setframe_ftsz(o, snap_slot(sn) != 0 ? (int32_t)*flinks-- : ftsz0); L->base = o+1; } } } lua_assert(map + nent == flinks); /* Compute current stack top. */ switch (bc_op(*pc)) { default: if (bc_op(*pc) < BC_FUNCF) { L->top = curr_topL(L); break; } /* fallthrough */ case BC_CALLM: case BC_CALLMT: case BC_RETM: case BC_TSETM: L->top = frame + snap->nslots; break; } return pc; }
/* Restore interpreter state from exit state with the help of a snapshot. */ const BCIns *lj_snap_restore(jit_State *J, void *exptr) { ExitState *ex = (ExitState *)exptr; SnapNo snapno = J->exitno; /* For now, snapno == exitno. */ GCtrace *T = traceref(J, J->parent); SnapShot *snap = &T->snap[snapno]; MSize n, nent = snap->nent; SnapEntry *map = &T->snapmap[snap->mapofs]; SnapEntry *flinks = map + nent + snap->depth; int32_t ftsz0; BCReg nslots = snap->nslots; TValue *frame; BloomFilter rfilt = snap_renamefilter(T, snapno); const BCIns *pc = snap_pc(map[nent]); lua_State *L = J->L; /* Set interpreter PC to the next PC to get correct error messages. */ setcframe_pc(cframe_raw(L->cframe), pc+1); /* Make sure the stack is big enough for the slots from the snapshot. */ if (LJ_UNLIKELY(L->base + nslots > tvref(L->maxstack))) { L->top = curr_topL(L); lj_state_growstack(L, nslots - curr_proto(L)->framesize); } /* Fill stack slots with data from the registers and spill slots. */ frame = L->base-1; ftsz0 = frame_ftsz(frame); /* Preserve link to previous frame in slot #0. */ for (n = 0; n < nent; n++) { SnapEntry sn = map[n]; IRRef ref = snap_ref(sn); BCReg s = snap_slot(sn); TValue *o = &frame[s]; /* Stack slots are relative to start frame. */ IRIns *ir = &T->ir[ref]; if (irref_isk(ref)) { /* Restore constant slot. */ lj_ir_kvalue(L, o, ir); if ((sn & (SNAP_CONT|SNAP_FRAME))) { /* Overwrite tag with frame link. */ o->fr.tp.ftsz = s != 0 ? (int32_t)*flinks-- : ftsz0; if ((sn & SNAP_FRAME)) { GCfunc *fn = ir_kfunc(ir); if (isluafunc(fn)) { MSize framesize = funcproto(fn)->framesize; L->base = ++o; if (LJ_UNLIKELY(o + framesize > tvref(L->maxstack))) { ptrdiff_t fsave = savestack(L, frame); L->top = o; lj_state_growstack(L, framesize); /* Grow again. */ frame = restorestack(L, fsave); } } } } } else if (!(sn & SNAP_NORESTORE)) { IRType1 t = ir->t; RegSP rs = ir->prev; lua_assert(!(sn & (SNAP_CONT|SNAP_FRAME))); if (LJ_UNLIKELY(bloomtest(rfilt, ref))) rs = snap_renameref(T, snapno, ref, rs); if (ra_hasspill(regsp_spill(rs))) { /* Restore from spill slot. */ int32_t *sps = &ex->spill[regsp_spill(rs)]; if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) { o->u32.lo = (uint32_t)*sps; } else if (irt_isinteger(t)) { setintV(o, *sps); #if !LJ_SOFTFP } else if (irt_isnum(t)) { o->u64 = *(uint64_t *)sps; #endif #if LJ_64 } else if (irt_islightud(t)) { /* 64 bit lightuserdata which may escape already has the tag bits. */ o->u64 = *(uint64_t *)sps; #endif } else { lua_assert(!irt_ispri(t)); /* PRI refs never have a spill slot. */ setgcrefi(o->gcr, *sps); setitype(o, irt_toitype(t)); } } else { /* Restore from register. */ Reg r = regsp_reg(rs); lua_assert(ra_hasreg(r)); if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) { o->u32.lo = (uint32_t)ex->gpr[r-RID_MIN_GPR]; } else if (irt_isinteger(t)) { setintV(o, (int32_t)ex->gpr[r-RID_MIN_GPR]); #if !LJ_SOFTFP } else if (irt_isnum(t)) { setnumV(o, ex->fpr[r-RID_MIN_FPR]); #endif #if LJ_64 } else if (irt_islightud(t)) { /* 64 bit lightuserdata which may escape already has the tag bits. */ o->u64 = ex->gpr[r-RID_MIN_GPR]; #endif } else { if (!irt_ispri(t)) setgcrefi(o->gcr, ex->gpr[r-RID_MIN_GPR]); setitype(o, irt_toitype(t)); } } if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) { rs = (ir+1)->prev; if (LJ_UNLIKELY(bloomtest(rfilt, ref+1))) rs = snap_renameref(T, snapno, ref+1, rs); o->u32.hi = (ra_hasspill(regsp_spill(rs))) ? (uint32_t)*&ex->spill[regsp_spill(rs)] : (uint32_t)ex->gpr[regsp_reg(rs)-RID_MIN_GPR]; } } } switch (bc_op(*pc)) { case BC_CALLM: case BC_CALLMT: case BC_RETM: case BC_TSETM: L->top = frame + nslots; break; default: L->top = curr_topL(L); break; } lua_assert(map + nent == flinks); return pc; }