/* Save current trace by copying and compacting it. */ static void trace_save(jit_State *J) { size_t sztr = ((sizeof(GCtrace)+7)&~7); size_t szins = (J->cur.nins-J->cur.nk)*sizeof(IRIns); size_t sz = sztr + szins + J->cur.nsnap*sizeof(SnapShot) + J->cur.nsnapmap*sizeof(SnapEntry); GCtrace *T = lj_mem_newt(J->L, (MSize)sz, GCtrace); char *p = (char *)T + sztr; memcpy(T, &J->cur, sizeof(GCtrace)); setgcrefr(T->nextgc, J2G(J)->gc.root); setgcrefp(J2G(J)->gc.root, T); newwhite(J2G(J), T); T->gct = ~LJ_TTRACE; T->ir = (IRIns *)p - J->cur.nk; memcpy(p, J->cur.ir+J->cur.nk, szins); p += szins; TRACE_APPENDVEC(snap, nsnap, SnapShot) TRACE_APPENDVEC(snapmap, nsnapmap, SnapEntry) J->cur.traceno = 0; setgcrefp(J->trace[T->traceno], T); lj_gc_barriertrace(J2G(J), T->traceno); lj_gdbjit_addtrace(J, T); #ifdef LUAJIT_USE_PERFTOOLS perftools_addtrace(T); #endif }
/* Flush all traces. */ int lj_trace_flushall(lua_State *L) { jit_State *J = L2J(L); ptrdiff_t i; if ((J2G(J)->hookmask & HOOK_GC)) return 1; for (i = (ptrdiff_t)J->sizetrace-1; i > 0; i--) { GCtrace *T = traceref(J, i); if (T) { if (T->root == 0) trace_flushroot(J, T); lj_gdbjit_deltrace(J, T); T->traceno = 0; setgcrefnull(J->trace[i]); } } J->cur.traceno = 0; J->freetrace = 0; /* Clear penalty cache. */ memset(J->penalty, 0, sizeof(J->penalty)); /* Free the whole machine code and invalidate all exit stub groups. */ lj_mcode_free(J); memset(J->exitstubgroup, 0, sizeof(J->exitstubgroup)); lj_vmevent_send(L, TRACE, setstrV(L, L->top++, lj_str_newlit(L, "flush")); );
/* Protection twiddling failed. Probably due to kernel security. */ static LJ_NOINLINE void mcode_protfail(jit_State *J) { lua_CFunction panic = J2G(J)->panic; if (panic) { lua_State *L = J->L; setstrV(L, L->top++, lj_err_str(L, LJ_ERR_JITPROT)); panic(L); } }
/* Save current trace by copying and compacting it. */ static void trace_save(jit_State *J, GCtrace *T) { size_t sztr = ((sizeof(GCtrace)+7)&~7); size_t szins = (J->cur.nins-J->cur.nk)*sizeof(IRIns); char *p = (char *)T + sztr; memcpy(T, &J->cur, sizeof(GCtrace)); setgcrefr(T->nextgc, J2G(J)->gc.root); setgcrefp(J2G(J)->gc.root, T); newwhite(J2G(J), T); T->gct = ~LJ_TTRACE; T->ir = (IRIns *)p - J->cur.nk; /* The IR has already been copied above. */ p += szins; TRACE_APPENDVEC(snap, nsnap, SnapShot) TRACE_APPENDVEC(snapmap, nsnapmap, SnapEntry) J->cur.traceno = 0; J->curfinal = NULL; setgcrefp(J->trace[T->traceno], T); lj_gc_barriertrace(J2G(J), T->traceno); lj_gdbjit_addtrace(J, T); #ifdef LUAJIT_USE_PERFTOOLS perftools_addtrace(T); #endif }
/* Get FILE* for I/O function. Any I/O error aborts recording, so there's ** no need to encode the alternate cases for any of the guards. */ static TRef recff_io_fp(jit_State *J, uint32_t id) { TRef tr, ud, fp; if (id) { /* io.func() */ tr = lj_ir_kptr(J, &J2G(J)->gcroot[id]); ud = emitir(IRT(IR_XLOAD, IRT_UDATA), tr, 0); } else { /* fp:method() */ ud = J->base[0]; if (!tref_isudata(ud)) lj_trace_err(J, LJ_TRERR_BADTYPE); tr = emitir(IRT(IR_FLOAD, IRT_U8), ud, IRFL_UDATA_UDTYPE); emitir(IRTGI(IR_EQ), tr, lj_ir_kint(J, UDTYPE_IO_FILE)); } fp = emitir(IRT(IR_FLOAD, IRT_PTR), ud, IRFL_UDATA_FILE); emitir(IRTG(IR_NE, IRT_PTR), fp, lj_ir_knull(J, IRT_PTR)); return fp; }
static TRef crec_tv_ct(jit_State *J, CType *s, CTypeID sid, TRef sp) { CTState *cts = ctype_ctsG(J2G(J)); CTInfo sinfo = s->info; lua_assert(!ctype_isenum(sinfo)); if (ctype_isnum(sinfo)) { IRType t = crec_ct2irt(s); TRef tr; if (t == IRT_CDATA) goto err_nyi; /* NYI: copyval of >64 bit integers. */ tr = emitir(IRT(IR_XLOAD, t), sp, 0); if (t == IRT_FLOAT || t == IRT_U32) { /* Keep uint32_t/float as numbers. */ tr = emitconv(tr, IRT_NUM, t, 0); } else if (t == IRT_I64 || t == IRT_U64) { /* Box 64 bit integer. */ TRef dp = emitir(IRTG(IR_CNEW, IRT_CDATA), lj_ir_kint(J, sid), TREF_NIL); TRef ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, sizeof(GCcdata))); emitir(IRT(IR_XSTORE, t), ptr, tr); return dp; } else if ((sinfo & CTF_BOOL)) { /* Assume not equal to zero. Fixup and emit pending guard later. */ lj_ir_set(J, IRTGI(IR_NE), tr, lj_ir_kint(J, 0)); J->postproc = LJ_POST_FIXGUARD; tr = TREF_TRUE; } return tr; } else if (ctype_isptr(sinfo)) { IRType t = (LJ_64 && s->size == 8) ? IRT_P64 : IRT_P32; sp = emitir(IRT(IR_XLOAD, t), sp, 0); } else if (ctype_isrefarray(sinfo) || ctype_isstruct(sinfo)) { cts->L = J->L; sid = lj_ctype_intern(cts, CTINFO_REF(sid), CTSIZE_PTR); /* Create ref. */ } else if (ctype_iscomplex(sinfo)) { /* Unbox/box complex. */ IRType t = s->size == 2*sizeof(double) ? IRT_NUM : IRT_FLOAT; ptrdiff_t esz = (ptrdiff_t)(s->size >> 1); TRef ptr, tr1, tr2, dp; dp = emitir(IRTG(IR_CNEW, IRT_CDATA), lj_ir_kint(J, sid), TREF_NIL); tr1 = emitir(IRT(IR_XLOAD, t), sp, 0); ptr = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, esz)); tr2 = emitir(IRT(IR_XLOAD, t), ptr, 0); ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, sizeof(GCcdata))); emitir(IRT(IR_XSTORE, t), ptr, tr1); ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, sizeof(GCcdata)+esz)); emitir(IRT(IR_XSTORE, t), ptr, tr2); return dp; } else {
/* Parse optimization parameter. */ static int jitopt_param(jit_State *J, const char *str) { const char *lst = JIT_P_STRING; int i; for (i = 0; i < JIT_P__MAX; i++) { size_t len = *(const uint8_t *)lst; TValue tv; lua_assert(len != 0); if (strncmp(str, lst+1, len) == 0 && str[len] == '=' && lj_str_numconv(&str[len+1], &tv)) { J->param[i] = lj_num2int(tv.n); if (i == JIT_P_hotloop) lj_dispatch_init_hotcount(J2G(J)); return 1; /* Ok. */ } lst += 1+len; } return 0; /* No match. */ }
/* Parse optimization parameter. */ static int jitopt_param(jit_State *J, const char *str) { const char *lst = JIT_P_STRING; int i; for (i = 0; i < JIT_P__MAX; i++) { size_t len = *(const uint8_t *)lst; lua_assert(len != 0); if (strncmp(str, lst+1, len) == 0 && str[len] == '=') { int32_t n = 0; const char *p = &str[len+1]; while (*p >= '0' && *p <= '9') n = n*10 + (*p++ - '0'); if (*p) return 0; /* Malformed number. */ J->param[i] = n; if (i == JIT_P_hotloop) lj_dispatch_init_hotcount(J2G(J)); return 1; /* Ok. */ } lst += 1+len; } return 0; /* No match. */ }
static CTypeID argv2ctype(jit_State *J, TRef tr, cTValue *o) { if (tref_isstr(tr)) { GCstr *s = strV(o); CPState cp; CTypeID oldtop; /* Specialize to the string containing the C type declaration. */ emitir(IRTG(IR_EQ, IRT_STR), tr, lj_ir_kstr(J, s)); cp.L = J->L; cp.cts = ctype_ctsG(J2G(J)); oldtop = cp.cts->top; cp.srcname = strdata(s); cp.p = strdata(s); cp.mode = CPARSE_MODE_ABSTRACT|CPARSE_MODE_NOIMPLICIT; if (lj_cparse(&cp) || cp.cts->top > oldtop) /* Avoid new struct defs. */ lj_trace_err(J, LJ_TRERR_BADTYPE); return cp.val.id; } else { GCcdata *cd = argv2cdata(J, tr, o); return cd->typeid == CTID_CTYPEID ? *(CTypeID *)cdataptr(cd) : cd->typeid; } }
static void mcode_free(jit_State *J, void *p, size_t sz) { lj_mem_free(J2G(J), p, sz); }
/* Unsink allocation from the trace exit state. Unsink sunk stores. */ static void snap_unsink(jit_State *J, GCtrace *T, ExitState *ex, SnapNo snapno, BloomFilter rfilt, IRIns *ir, TValue *o) { lua_assert(ir->o == IR_TNEW || ir->o == IR_TDUP || ir->o == IR_CNEW || ir->o == IR_CNEWI); #if LJ_HASFFI if (ir->o == IR_CNEW || ir->o == IR_CNEWI) { CTState *cts = ctype_ctsG(J2G(J)); CTypeID id = (CTypeID)T->ir[ir->op1].i; CTSize sz = lj_ctype_size(cts, id); GCcdata *cd = lj_cdata_new(cts, id, sz); setcdataV(J->L, o, cd); if (ir->o == IR_CNEWI) { uint8_t *p = (uint8_t *)cdataptr(cd); lua_assert(sz == 4 || sz == 8); if (LJ_32 && sz == 8 && ir+1 < T->ir + T->nins && (ir+1)->o == IR_HIOP) { snap_restoredata(T, ex, snapno, rfilt, (ir+1)->op2, LJ_LE?p+4:p, 4); if (LJ_BE) p += 4; sz = 4; } snap_restoredata(T, ex, snapno, rfilt, ir->op2, p, sz); } else { IRIns *irs, *irlast = &T->ir[T->snap[snapno].ref]; for (irs = ir+1; irs < irlast; irs++) if (irs->r == RID_SINK && snap_sunk_store(J, ir, irs)) { IRIns *iro = &T->ir[T->ir[irs->op1].op2]; uint8_t *p = (uint8_t *)cd; CTSize szs; lua_assert(irs->o == IR_XSTORE && T->ir[irs->op1].o == IR_ADD); lua_assert(iro->o == IR_KINT || iro->o == IR_KINT64); if (irt_is64(irs->t)) szs = 8; else if (irt_isi8(irs->t) || irt_isu8(irs->t)) szs = 1; else if (irt_isi16(irs->t) || irt_isu16(irs->t)) szs = 2; else szs = 4; if (LJ_64 && iro->o == IR_KINT64) p += (int64_t)ir_k64(iro)->u64; else p += iro->i; lua_assert(p >= (uint8_t *)cdataptr(cd) && p + szs <= (uint8_t *)cdataptr(cd) + sz); if (LJ_32 && irs+1 < T->ir + T->nins && (irs+1)->o == IR_HIOP) { lua_assert(szs == 4); snap_restoredata(T, ex, snapno, rfilt, (irs+1)->op2, LJ_LE?p+4:p,4); if (LJ_BE) p += 4; } snap_restoredata(T, ex, snapno, rfilt, irs->op2, p, szs); } } } else #endif { IRIns *irs, *irlast; GCtab *t = ir->o == IR_TNEW ? lj_tab_new(J->L, ir->op1, ir->op2) : lj_tab_dup(J->L, ir_ktab(&T->ir[ir->op1])); settabV(J->L, o, t); irlast = &T->ir[T->snap[snapno].ref]; for (irs = ir+1; irs < irlast; irs++) if (irs->r == RID_SINK && snap_sunk_store(J, ir, irs)) { IRIns *irk = &T->ir[irs->op1]; TValue tmp, *val; lua_assert(irs->o == IR_ASTORE || irs->o == IR_HSTORE || irs->o == IR_FSTORE); if (irk->o == IR_FREF) { lua_assert(irk->op2 == IRFL_TAB_META); snap_restoreval(J, T, ex, snapno, rfilt, irs->op2, &tmp); /* NOBARRIER: The table is new (marked white). */ setgcref(t->metatable, obj2gco(tabV(&tmp))); } else { irk = &T->ir[irk->op2]; if (irk->o == IR_KSLOT) irk = &T->ir[irk->op1]; lj_ir_kvalue(J->L, &tmp, irk); val = lj_tab_set(J->L, t, &tmp); /* NOBARRIER: The table is new (marked white). */ snap_restoreval(J, T, ex, snapno, rfilt, irs->op2, val); if (LJ_SOFTFP && irs+1 < T->ir + T->nins && (irs+1)->o == IR_HIOP) { snap_restoreval(J, T, ex, snapno, rfilt, (irs+1)->op2, &tmp); val->u32.hi = tmp.u32.lo; } } } } }