static void unlinkuv(GCupval *uv) { lua_assert(uvprev(uvnext(uv)) == uv && uvnext(uvprev(uv)) == uv); setgcrefr(uvnext(uv)->prev, uv->prev); setgcrefr(uvprev(uv)->next, uv->next); }
static void unlinkupval (UpVal *uv) { lua_assert(uv->u.l.next->u.l.prev == uv && uv->u.l.prev->u.l.next == uv); uv->u.l.next->u.l.prev = uv->u.l.prev; /* remove from `uvhead' list */ uv->u.l.prev->u.l.next = uv->u.l.next; }
void luaG_concaterror (lua_State *L, StkId p1, StkId p2) { if (ttisstring(p1) || ttisnumber(p1)) p1 = p2; lua_assert(!ttisstring(p1) && !ttisnumber(p1)); luaG_typeerror(L, p1, "concatenate"); }
static void removeentry (Node *n) { lua_assert(ttisnil(gval(n))); if (iscollectable(gkey(n))) setttype(gkey(n), LUA_TDEADKEY); /* dead key; remove it */ }
void ravi_emit_iFORPREP(ravi_function_def_t *def, int A, int pc, int step_one) { ravi_branch_def_t *forloop_target = def->jmp_targets[pc]; assert(forloop_target); forloop_target->ilimit = gcc_jit_function_new_local( def->jit_function, NULL, def->ravi->types->lua_IntegerT, unique_name(def, "ilimit", 0)); if (!step_one) { forloop_target->istep = gcc_jit_function_new_local( def->jit_function, NULL, def->ravi->types->lua_IntegerT, unique_name(def, "istep", 0)); } forloop_target->iidx = gcc_jit_function_new_local( def->jit_function, NULL, def->ravi->types->lua_IntegerT, unique_name(def, "iidx", 0)); // lua_Integer initv = ivalue(init); // setivalue(init, initv - ivalue(pstep)); // Load pointer to base ravi_emit_load_base(def); // TValue *init = ra; // TValue *pstep = ra + 2; gcc_jit_lvalue *init = ravi_emit_get_register(def, A); gcc_jit_lvalue *plimit = ravi_emit_get_register(def, A + 1); gcc_jit_lvalue *pstep = NULL; if (!step_one) pstep = ravi_emit_get_register(def, A + 2); // Get ivalue(pstep) gcc_jit_lvalue *limit_ivalue = ravi_emit_load_reg_i(def, plimit); gcc_jit_lvalue *init_ivalue = ravi_emit_load_reg_i(def, init); if (!step_one) { // setivalue(init, initv - ivalue(pstep)); gcc_jit_lvalue *step_ivalue = ravi_emit_load_reg_i(def, pstep); gcc_jit_rvalue *idx = gcc_jit_context_new_binary_op( def->function_context, NULL, GCC_JIT_BINARY_OP_MINUS, def->ravi->types->lua_IntegerT, gcc_jit_lvalue_as_rvalue(init_ivalue), gcc_jit_lvalue_as_rvalue(step_ivalue)); // Save idx gcc_jit_block_add_assignment(def->current_block, NULL, forloop_target->iidx, idx); // Save step gcc_jit_block_add_assignment(def->current_block, NULL, forloop_target->istep, gcc_jit_lvalue_as_rvalue(step_ivalue)); } else { // setivalue(init, initv - ivalue(pstep)); gcc_jit_rvalue *idx = gcc_jit_context_new_binary_op( def->function_context, NULL, GCC_JIT_BINARY_OP_MINUS, def->ravi->types->lua_IntegerT, gcc_jit_lvalue_as_rvalue(init_ivalue), gcc_jit_context_new_rvalue_from_int(def->function_context, def->ravi->types->lua_IntegerT, 1)); // Save idx gcc_jit_block_add_assignment(def->current_block, NULL, forloop_target->iidx, idx); } // Save limit gcc_jit_block_add_assignment(def->current_block, NULL, forloop_target->ilimit, gcc_jit_lvalue_as_rvalue(limit_ivalue)); // We are done so jump to forloop lua_assert(def->jmp_targets[pc]->jmp); ravi_emit_branch(def, def->jmp_targets[pc]->jmp); }
/* ** if key is not marked, mark its entry as dead (therefore removing it ** from the table) */ static void removeentry (Node *n) { lua_assert(ttisnil(gval(n))); if (valiswhite(gkey(n))) setdeadvalue(gkey(n)); /* unused and unmarked key; remove it */ }
static int codeextraarg (FuncState *fs, int a) { lua_assert(a <= MAXARG_Ax); return luaK_code(fs, CREATE_Ax(OP_EXTRAARG, a)); }
/* Replay snapshot state to setup side trace. */ void lj_snap_replay(jit_State *J, GCtrace *T) { SnapShot *snap = &T->snap[J->exitno]; SnapEntry *map = &T->snapmap[snap->mapofs]; MSize n, nent = snap->nent; BloomFilter seen = 0; int pass23 = 0; J->framedepth = 0; /* Emit IR for slots inherited from parent snapshot. */ for (n = 0; n < nent; n++) { SnapEntry sn = map[n]; BCReg s = snap_slot(sn); IRRef ref = snap_ref(sn); IRIns *ir = &T->ir[ref]; TRef tr; /* The bloom filter avoids O(nent^2) overhead for de-duping slots. */ if (bloomtest(seen, ref) && (tr = snap_dedup(J, map, n, ref)) != 0) goto setslot; bloomset(seen, ref); if (irref_isk(ref)) { tr = snap_replay_const(J, ir); } else if (!regsp_used(ir->prev)) { pass23 = 1; lua_assert(s != 0); tr = s; } else { IRType t = irt_type(ir->t); uint32_t mode = IRSLOAD_INHERIT|IRSLOAD_PARENT; if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) t = IRT_NUM; if (ir->o == IR_SLOAD) mode |= (ir->op2 & IRSLOAD_READONLY); tr = emitir_raw(IRT(IR_SLOAD, t), s, mode); } setslot: J->slot[s] = tr | (sn&(SNAP_CONT|SNAP_FRAME)); /* Same as TREF_* flags. */ J->framedepth += ((sn & (SNAP_CONT|SNAP_FRAME)) && s); if ((sn & SNAP_FRAME)) J->baseslot = s+1; } if (pass23) { IRIns *irlast = &T->ir[snap->ref]; pass23 = 0; /* Emit dependent PVALs. */ for (n = 0; n < nent; n++) { SnapEntry sn = map[n]; IRRef refp = snap_ref(sn); IRIns *ir = &T->ir[refp]; if (regsp_reg(ir->r) == RID_SUNK) { if (J->slot[snap_slot(sn)] != snap_slot(sn)) continue; pass23 = 1; lua_assert(ir->o == IR_TNEW || ir->o == IR_TDUP || ir->o == IR_CNEW || ir->o == IR_CNEWI); if (ir->op1 >= T->nk) snap_pref(J, T, map, nent, seen, ir->op1); if (ir->op2 >= T->nk) snap_pref(J, T, map, nent, seen, ir->op2); if (LJ_HASFFI && ir->o == IR_CNEWI) { if (LJ_32 && refp+1 < T->nins && (ir+1)->o == IR_HIOP) snap_pref(J, T, map, nent, seen, (ir+1)->op2); } else { IRIns *irs; for (irs = ir+1; irs < irlast; irs++) if (irs->r == RID_SINK && snap_sunk_store(T, ir, irs)) { if (snap_pref(J, T, map, nent, seen, irs->op2) == 0) snap_pref(J, T, map, nent, seen, T->ir[irs->op2].op1); else if ((LJ_SOFTFP || (LJ_32 && LJ_HASFFI)) && irs+1 < irlast && (irs+1)->o == IR_HIOP) snap_pref(J, T, map, nent, seen, (irs+1)->op2); } } } else if (!irref_isk(refp) && !regsp_used(ir->prev)) { lua_assert(ir->o == IR_CONV && ir->op2 == IRCONV_NUM_INT); J->slot[snap_slot(sn)] = snap_pref(J, T, map, nent, seen, ir->op1); } } /* Replay sunk instructions. */ for (n = 0; pass23 && n < nent; n++) { SnapEntry sn = map[n]; IRRef refp = snap_ref(sn); IRIns *ir = &T->ir[refp]; if (regsp_reg(ir->r) == RID_SUNK) { TRef op1, op2; if (J->slot[snap_slot(sn)] != snap_slot(sn)) { /* De-dup allocs. */ J->slot[snap_slot(sn)] = J->slot[J->slot[snap_slot(sn)]]; continue; } op1 = ir->op1; if (op1 >= T->nk) op1 = snap_pref(J, T, map, nent, seen, op1); op2 = ir->op2; if (op2 >= T->nk) op2 = snap_pref(J, T, map, nent, seen, op2); if (LJ_HASFFI && ir->o == IR_CNEWI) { if (LJ_32 && refp+1 < T->nins && (ir+1)->o == IR_HIOP) { lj_needsplit(J); /* Emit joining HIOP. */ op2 = emitir_raw(IRT(IR_HIOP, IRT_I64), op2, snap_pref(J, T, map, nent, seen, (ir+1)->op2)); } J->slot[snap_slot(sn)] = emitir(ir->ot & ~(IRT_MARK|IRT_ISPHI), op1, op2); } else { IRIns *irs; TRef tr = emitir(ir->ot, op1, op2); J->slot[snap_slot(sn)] = tr; for (irs = ir+1; irs < irlast; irs++) if (irs->r == RID_SINK && snap_sunk_store(T, ir, irs)) { IRIns *irr = &T->ir[irs->op1]; TRef val, key = irr->op2, tmp = tr; if (irr->o != IR_FREF) { IRIns *irk = &T->ir[key]; if (irr->o == IR_HREFK) key = lj_ir_kslot(J, snap_replay_const(J, &T->ir[irk->op1]), irk->op2); else key = snap_replay_const(J, irk); if (irr->o == IR_HREFK || irr->o == IR_AREF) { IRIns *irf = &T->ir[irr->op1]; tmp = emitir(irf->ot, tmp, irf->op2); } } tmp = emitir(irr->ot, tmp, key); val = snap_pref(J, T, map, nent, seen, irs->op2); if (val == 0) { IRIns *irc = &T->ir[irs->op2]; lua_assert(irc->o == IR_CONV && irc->op2 == IRCONV_NUM_INT); val = snap_pref(J, T, map, nent, seen, irc->op1); val = emitir(IRTN(IR_CONV), val, IRCONV_NUM_INT); } else if ((LJ_SOFTFP || (LJ_32 && LJ_HASFFI)) && irs+1 < irlast && (irs+1)->o == IR_HIOP) { IRType t = IRT_I64; if (LJ_SOFTFP && irt_type((irs+1)->t) == IRT_SOFTFP) t = IRT_NUM; lj_needsplit(J); if (irref_isk(irs->op2) && irref_isk((irs+1)->op2)) { uint64_t k = (uint32_t)T->ir[irs->op2].i + ((uint64_t)T->ir[(irs+1)->op2].i << 32); val = lj_ir_k64(J, t == IRT_I64 ? IR_KINT64 : IR_KNUM, lj_ir_k64_find(J, k)); } else { val = emitir_raw(IRT(IR_HIOP, t), val, snap_pref(J, T, map, nent, seen, (irs+1)->op2)); } tmp = emitir(IRT(irs->o, t), tmp, val); continue; } tmp = emitir(irs->ot, tmp, val); } else if (LJ_HASFFI && irs->o == IR_XBAR && ir->o == IR_CNEW) { emitir(IRT(IR_XBAR, IRT_NIL), 0, 0); } } } } } J->base = J->slot + J->baseslot; J->maxslot = snap->nslots - J->baseslot; lj_snap_add(J); if (pass23) /* Need explicit GC step _after_ initial snapshot. */ emitir_raw(IRTG(IR_GCSTEP, IRT_NIL), 0, 0); }
/* Unsink allocation from the trace exit state. Unsink sunk stores. */ static void snap_unsink(jit_State *J, GCtrace *T, ExitState *ex, SnapNo snapno, BloomFilter rfilt, IRIns *ir, TValue *o) { lua_assert(ir->o == IR_TNEW || ir->o == IR_TDUP || ir->o == IR_CNEW || ir->o == IR_CNEWI); #if LJ_HASFFI if (ir->o == IR_CNEW || ir->o == IR_CNEWI) { CTState *cts = ctype_cts(J->L); CTypeID id = (CTypeID)T->ir[ir->op1].i; CTSize sz = lj_ctype_size(cts, id); GCcdata *cd = lj_cdata_new(cts, id, sz); setcdataV(J->L, o, cd); if (ir->o == IR_CNEWI) { uint8_t *p = (uint8_t *)cdataptr(cd); lua_assert(sz == 4 || sz == 8); if (LJ_32 && sz == 8 && ir+1 < T->ir + T->nins && (ir+1)->o == IR_HIOP) { snap_restoredata(T, ex, snapno, rfilt, (ir+1)->op2, LJ_LE?p+4:p, 4); if (LJ_BE) p += 4; sz = 4; } snap_restoredata(T, ex, snapno, rfilt, ir->op2, p, sz); } else { IRIns *irs, *irlast = &T->ir[T->snap[snapno].ref]; for (irs = ir+1; irs < irlast; irs++) if (irs->r == RID_SINK && snap_sunk_store(T, ir, irs)) { IRIns *iro = &T->ir[T->ir[irs->op1].op2]; uint8_t *p = (uint8_t *)cd; CTSize szs; lua_assert(irs->o == IR_XSTORE && T->ir[irs->op1].o == IR_ADD); lua_assert(iro->o == IR_KINT || iro->o == IR_KINT64); if (irt_is64(irs->t)) szs = 8; else if (irt_isi8(irs->t) || irt_isu8(irs->t)) szs = 1; else if (irt_isi16(irs->t) || irt_isu16(irs->t)) szs = 2; else szs = 4; if (LJ_64 && iro->o == IR_KINT64) p += (int64_t)ir_k64(iro)->u64; else p += iro->i; lua_assert(p >= (uint8_t *)cdataptr(cd) && p + szs <= (uint8_t *)cdataptr(cd) + sz); if (LJ_32 && irs+1 < T->ir + T->nins && (irs+1)->o == IR_HIOP) { lua_assert(szs == 4); snap_restoredata(T, ex, snapno, rfilt, (irs+1)->op2, LJ_LE?p+4:p,4); if (LJ_BE) p += 4; } snap_restoredata(T, ex, snapno, rfilt, irs->op2, p, szs); } } } else #endif { IRIns *irs, *irlast; GCtab *t = ir->o == IR_TNEW ? lj_tab_new(J->L, ir->op1, ir->op2) : lj_tab_dup(J->L, ir_ktab(&T->ir[ir->op1])); settabV(J->L, o, t); irlast = &T->ir[T->snap[snapno].ref]; for (irs = ir+1; irs < irlast; irs++) if (irs->r == RID_SINK && snap_sunk_store(T, ir, irs)) { IRIns *irk = &T->ir[irs->op1]; TValue tmp, *val; lua_assert(irs->o == IR_ASTORE || irs->o == IR_HSTORE || irs->o == IR_FSTORE); if (irk->o == IR_FREF) { lua_assert(irk->op2 == IRFL_TAB_META); snap_restoreval(J, T, ex, snapno, rfilt, irs->op2, &tmp); /* NOBARRIER: The table is new (marked white). */ setgcref(t->metatable, obj2gco(tabV(&tmp))); } else { irk = &T->ir[irk->op2]; if (irk->o == IR_KSLOT) irk = &T->ir[irk->op1]; lj_ir_kvalue(J->L, &tmp, irk); val = lj_tab_set(J->L, t, &tmp); /* NOBARRIER: The table is new (marked white). */ snap_restoreval(J, T, ex, snapno, rfilt, irs->op2, val); if (LJ_SOFTFP && irs+1 < T->ir + T->nins && (irs+1)->o == IR_HIOP) { snap_restoreval(J, T, ex, snapno, rfilt, (irs+1)->op2, &tmp); val->u32.hi = tmp.u32.lo; } } } } }
/* Top-level delegating persist function */ static void persist(PersistInfo *pi) { /* Grab the object's type. */ int type = lua_type(pi->L, -1); int simple = type == LUA_TNIL || type == LUA_TBOOLEAN || type == LUA_TNUMBER || type == LUA_TLIGHTUSERDATA; /* Increment the number of objects persisted. */ ++(pi->counter); /* If the type isn't simple, check to see if one is already in the reftable. */ if(!simple) { /* perms reftbl ... obj */ lua_checkstack(pi->L, 2); /* If the object has already been written, write a reference to it */ lua_pushvalue(pi->L, -1); /* perms reftbl ... obj obj */ lua_rawget(pi->L, 2); /* perms reftbl ... obj ref? */ if(!lua_isnil(pi->L, -1)) { /* perms reftbl ... obj ref */ int zero = 0; int ref = (intptr_t)lua_touserdata(pi->L, -1); pi->writer(pi->L, &zero, sizeof(int), pi->ud); pi->writer(pi->L, &ref, sizeof(int), pi->ud); lua_pop(pi->L, 1); /* perms reftbl ... obj ref */ #ifdef PLUTO_DEBUG printindent(pi->level); printf("0 %d\n", ref); #endif return; } /* perms reftbl ... obj nil */ lua_pop(pi->L, 1); } /* perms reftbl ... obj */ /* If the object is nil, write the pseudoreference 0 */ if(lua_isnil(pi->L, -1)) { int zero = 0; /* firsttime */ pi->writer(pi->L, &zero, sizeof(int), pi->ud); /* ref */ pi->writer(pi->L, &zero, sizeof(int), pi->ud); #ifdef PLUTO_DEBUG printindent(pi->level); printf("0 0\n"); #endif return; } { /* indicate that it's the first time */ int one = 1; pi->writer(pi->L, &one, sizeof(int), pi->ud); } /* put the value in the reftable if necessary. Simple types don't need to be put in the reftable. */ if(!simple) { lua_pushvalue(pi->L, -1); /* perms reftbl ... obj obj */ lua_pushlightuserdata(pi->L, (void*)((intptr_t) pi->counter)); /* perms reftbl ... obj obj ref */ lua_rawset(pi->L, 2); /* perms reftbl ... obj */ } pi->writer(pi->L, &pi->counter, sizeof(int), pi->ud); /* At this point, we'll give the permanents table a chance to play. */ { lua_pushvalue(pi->L, -1); /* perms reftbl ... obj obj */ lua_gettable(pi->L, 1); /* perms reftbl ... obj permkey? */ if(!lua_isnil(pi->L, -1)) { /* perms reftbl ... obj permkey */ int type = PLUTO_TPERMANENT; #ifdef PLUTO_DEBUG printindent(pi->level); printf("1 %d PERM\n", pi->counter); pi->level++; #endif pi->writer(pi->L, &type, sizeof(int), pi->ud); persist(pi); lua_pop(pi->L, 1); /* perms reftbl ... obj */ #ifdef PLUTO_DEBUG pi->level--; #endif return; } else { /* perms reftbl ... obj nil */ lua_pop(pi->L, 1); /* perms reftbl ... obj */ } /* perms reftbl ... obj */ } { int type = lua_type(pi->L, -1); pi->writer(pi->L, &type, sizeof(int), pi->ud); #ifdef PLUTO_DEBUG printindent(pi->level); printf("1 %d %d\n", pi->counter, type); pi->level++; #endif } switch(lua_type(pi->L, -1)) { case LUA_TBOOLEAN: persistboolean(pi); break; case LUA_TLIGHTUSERDATA: persistlightuserdata(pi); break; case LUA_TNUMBER: persistnumber(pi); break; case LUA_TSTRING: persiststring(pi); break; case LUA_TTABLE: persisttable(pi); break; case LUA_TFUNCTION: persistfunction(pi); break; case LUA_TTHREAD: persistthread(pi); break; case LUA_TPROTO: persistproto(pi); break; case LUA_TUPVAL: persistupval(pi); break; case LUA_TUSERDATA: persistuserdata(pi); break; default: lua_assert(0); } #ifdef PLUTO_DEBUG pi->level--; #endif }
/* Find unused slots with reaching-definitions bytecode data-flow analysis. */ static BCReg snap_usedef(jit_State *J, uint8_t *udf, const BCIns *pc, BCReg maxslot) { BCReg s; GCobj *o; if (maxslot == 0) return 0; #ifdef LUAJIT_USE_VALGRIND /* Avoid errors for harmless reads beyond maxslot. */ memset(udf, 1, SNAP_USEDEF_SLOTS); #else memset(udf, 1, maxslot); #endif /* Treat open upvalues as used. */ o = gcref(J->L->openupval); while (o) { if (uvval(gco2uv(o)) < J->L->base) break; udf[uvval(gco2uv(o)) - J->L->base] = 0; o = gcref(o->gch.nextgc); } #define USE_SLOT(s) udf[(s)] &= ~1 #define DEF_SLOT(s) udf[(s)] *= 3 /* Scan through following bytecode and check for uses/defs. */ lua_assert(pc >= proto_bc(J->pt) && pc < proto_bc(J->pt) + J->pt->sizebc); for (;;) { BCIns ins = *pc++; BCOp op = bc_op(ins); switch (bcmode_b(op)) { case BCMvar: USE_SLOT(bc_b(ins)); break; default: break; } switch (bcmode_c(op)) { case BCMvar: USE_SLOT(bc_c(ins)); break; case BCMrbase: lua_assert(op == BC_CAT); for (s = bc_b(ins); s <= bc_c(ins); s++) USE_SLOT(s); for (; s < maxslot; s++) DEF_SLOT(s); break; case BCMjump: handle_jump: { BCReg minslot = bc_a(ins); if (op >= BC_FORI && op <= BC_JFORL) minslot += FORL_EXT; else if (op >= BC_ITERL && op <= BC_JITERL) minslot += bc_b(pc[-2])-1; else if (op == BC_UCLO) { pc += bc_j(ins); break; } for (s = minslot; s < maxslot; s++) DEF_SLOT(s); return minslot < maxslot ? minslot : maxslot; } case BCMlit: if (op == BC_JFORL || op == BC_JITERL || op == BC_JLOOP) { goto handle_jump; } else if (bc_isret(op)) { BCReg top = op == BC_RETM ? maxslot : (bc_a(ins) + bc_d(ins)-1); for (s = 0; s < bc_a(ins); s++) DEF_SLOT(s); for (; s < top; s++) USE_SLOT(s); for (; s < maxslot; s++) DEF_SLOT(s); return 0; } break; case BCMfunc: return maxslot; /* NYI: will abort, anyway. */ default: break; } switch (bcmode_a(op)) { case BCMvar: USE_SLOT(bc_a(ins)); break; case BCMdst: if (!(op == BC_ISTC || op == BC_ISFC)) DEF_SLOT(bc_a(ins)); break; case BCMbase: if (op >= BC_CALLM && op <= BC_VARG) { BCReg top = (op == BC_CALLM || op == BC_CALLMT || bc_c(ins) == 0) ? maxslot : (bc_a(ins) + bc_c(ins)+LJ_FR2); if (LJ_FR2) DEF_SLOT(bc_a(ins)+1); s = bc_a(ins) - ((op == BC_ITERC || op == BC_ITERN) ? 3 : 0); for (; s < top; s++) USE_SLOT(s); for (; s < maxslot; s++) DEF_SLOT(s); if (op == BC_CALLT || op == BC_CALLMT) { for (s = 0; s < bc_a(ins); s++) DEF_SLOT(s); return 0; } } else if (op == BC_KNIL) { for (s = bc_a(ins); s <= bc_d(ins); s++) DEF_SLOT(s); } else if (op == BC_TSETM) { for (s = bc_a(ins)-1; s < maxslot; s++) USE_SLOT(s); } break; default: break; } lua_assert(pc >= proto_bc(J->pt) && pc < proto_bc(J->pt) + J->pt->sizebc); } #undef USE_SLOT #undef DEF_SLOT return 0; /* unreachable */ }
/* Persist all stack members */ static void persistthread(PersistInfo *pi) { size_t posremaining; lua_State *L2; /* perms reftbl ... thr */ L2 = lua_tothread(pi->L, -1); lua_checkstack(pi->L, L2->top - L2->stack + 1); if(pi->L == L2) { lua_pushstring(pi->L, "Can't persist currently running thread"); lua_error(pi->L); return; /* not reached */ } /* Persist the stack */ posremaining = revappendstack(L2, pi->L); /* perms reftbl ... thr (rev'ed contents of L2) */ pi->writer(pi->L, &posremaining, sizeof(size_t), pi->ud); for(; posremaining > 0; posremaining--) { persist(pi); lua_pop(pi->L, 1); } /* perms reftbl ... thr */ /* Now, persist the CallInfo stack. */ { size_t i, numframes = (L2->ci - L2->base_ci) + 1; pi->writer(pi->L, &numframes, sizeof(size_t), pi->ud); for(i=0; i<numframes; i++) { CallInfo *ci = L2->base_ci + i; size_t stackbase = ci->base - L2->stack; size_t stackfunc = ci->func - L2->stack; size_t stacktop = ci->top - L2->stack; size_t savedpc = (ci != L2->base_ci) ? ci->savedpc - ci_func(ci)->l.p->code : 0; pi->writer(pi->L, &stackbase, sizeof(size_t), pi->ud); pi->writer(pi->L, &stackfunc, sizeof(size_t), pi->ud); pi->writer(pi->L, &stacktop, sizeof(size_t), pi->ud); pi->writer(pi->L, &ci->nresults, sizeof(int), pi->ud); pi->writer(pi->L, &savedpc, sizeof(size_t), pi->ud); } } /* Serialize the state's other parameters, with the exception of upval stuff */ { size_t stackbase = L2->base - L2->stack; size_t stacktop = L2->top - L2->stack; lua_assert(L2->nCcalls <= 1); pi->writer(pi->L, &L2->status, sizeof(lu_byte), pi->ud); pi->writer(pi->L, &stackbase, sizeof(size_t), pi->ud); pi->writer(pi->L, &stacktop, sizeof(size_t), pi->ud); pi->writer(pi->L, &L2->errfunc, sizeof(ptrdiff_t), pi->ud); } /* Finally, record upvalues which need to be reopened */ /* See the comment above persistupval() for why we do this */ { GCObject *gco; UpVal *uv; /* perms reftbl ... thr */ for(gco = L2->openupval; gco != NULL; gco = uv->next) { size_t stackpos; uv = gco2uv(gco); /* Make sure upvalue is really open */ lua_assert(uv->v != &uv->u.value); pushupval(pi->L, uv); /* perms reftbl ... thr uv */ persist(pi); lua_pop(pi->L, 1); /* perms reftbl ... thr */ stackpos = uv->v - L2->stack; pi->writer(pi->L, &stackpos, sizeof(size_t), pi->ud); } /* perms reftbl ... thr */ lua_pushnil(pi->L); /* perms reftbl ... thr nil */ persist(pi); lua_pop(pi->L, 1); /* perms reftbl ... thr */ } /* perms reftbl ... thr */ }
static UpVal *toupval(lua_State *L, int stackpos) { lua_assert(ttype(getobject(L, stackpos)) == LUA_TUPVAL); return gco2uv(getobject(L, stackpos)->value.gc); }
void rstack_close (lua_State *L) { RStack *rs = rstack(L); lua_assert(rs->cregnum == 0); lua_assert(rs->creg->top == rs->rbuf.head); buf_resize(L, rs, 0); }
static LocVar *getlocvar (FuncState *fs, int i) { int idx = fs->ls->dyd->actvar.arr[fs->firstlocal + i].idx; lua_assert(idx < fs->nlocvars); return &fs->f->locvars[idx]; }
/* Restore interpreter state from exit state with the help of a snapshot. */ const BCIns *lj_snap_restore(jit_State *J, void *exptr) { ExitState *ex = (ExitState *)exptr; SnapNo snapno = J->exitno; /* For now, snapno == exitno. */ GCtrace *T = traceref(J, J->parent); SnapShot *snap = &T->snap[snapno]; MSize n, nent = snap->nent; SnapEntry *map = &T->snapmap[snap->mapofs]; SnapEntry *flinks = &T->snapmap[snap_nextofs(T, snap)-1]; ptrdiff_t ftsz0; TValue *frame; BloomFilter rfilt = snap_renamefilter(T, snapno); const BCIns *pc = snap_pc(map[nent]); lua_State *L = J->L; /* Set interpreter PC to the next PC to get correct error messages. */ setcframe_pc(cframe_raw(L->cframe), pc+1); /* Make sure the stack is big enough for the slots from the snapshot. */ if (LJ_UNLIKELY(L->base + snap->topslot >= tvref(L->maxstack))) { L->top = curr_topL(L); lj_state_growstack(L, snap->topslot - curr_proto(L)->framesize); } /* Fill stack slots with data from the registers and spill slots. */ frame = L->base-1; ftsz0 = frame_ftsz(frame); /* Preserve link to previous frame in slot #0. */ for (n = 0; n < nent; n++) { SnapEntry sn = map[n]; if (!(sn & SNAP_NORESTORE)) { TValue *o = &frame[snap_slot(sn)]; IRRef ref = snap_ref(sn); IRIns *ir = &T->ir[ref]; if (ir->r == RID_SUNK) { MSize j; for (j = 0; j < n; j++) if (snap_ref(map[j]) == ref) { /* De-duplicate sunk allocations. */ copyTV(L, o, &frame[snap_slot(map[j])]); goto dupslot; } snap_unsink(J, T, ex, snapno, rfilt, ir, o); dupslot: continue; } snap_restoreval(J, T, ex, snapno, rfilt, ref, o); if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM) && tvisint(o)) { TValue tmp; snap_restoreval(J, T, ex, snapno, rfilt, ref+1, &tmp); o->u32.hi = tmp.u32.lo; } else if ((sn & (SNAP_CONT|SNAP_FRAME))) { lua_assert(!LJ_FR2); /* TODO_FR2: store 64 bit PCs. */ /* Overwrite tag with frame link. */ setframe_ftsz(o, snap_slot(sn) != 0 ? (int32_t)*flinks-- : ftsz0); L->base = o+1; } } } lua_assert(map + nent == flinks); /* Compute current stack top. */ switch (bc_op(*pc)) { default: if (bc_op(*pc) < BC_FUNCF) { L->top = curr_topL(L); break; } /* fallthrough */ case BC_CALLM: case BC_CALLMT: case BC_RETM: case BC_TSETM: L->top = frame + snap->nslots; break; } return pc; }
/* Index C data by a TValue. Return CType and pointer. */ CType *lj_cdata_index(CTState *cts, GCcdata *cd, cTValue *key, uint8_t **pp, CTInfo *qual) { uint8_t *p = (uint8_t *)cdataptr(cd); CType *ct = ctype_get(cts, cd->ctypeid); ptrdiff_t idx; /* Resolve reference for cdata object. */ if (ctype_isref(ct->info)) { lua_assert(ct->size == CTSIZE_PTR); p = *(uint8_t **)p; ct = ctype_child(cts, ct); } collect_attrib: /* Skip attributes and collect qualifiers. */ while (ctype_isattrib(ct->info)) { if (ctype_attrib(ct->info) == CTA_QUAL) *qual |= ct->size; ct = ctype_child(cts, ct); } lua_assert(!ctype_isref(ct->info)); /* Interning rejects refs to refs. */ if (tvisint(key)) { idx = (ptrdiff_t)intV(key); goto integer_key; } else if (tvisnum(key)) { /* Numeric key. */ idx = LJ_64 ? (ptrdiff_t)numV(key) : (ptrdiff_t)lj_num2int(numV(key)); integer_key: if (ctype_ispointer(ct->info)) { CTSize sz = lj_ctype_size(cts, ctype_cid(ct->info)); /* Element size. */ if (sz == CTSIZE_INVALID) lj_err_caller(cts->L, LJ_ERR_FFI_INVSIZE); if (ctype_isptr(ct->info)) { p = (uint8_t *)cdata_getptr(p, ct->size); } else if ((ct->info & (CTF_VECTOR|CTF_COMPLEX))) { if ((ct->info & CTF_COMPLEX)) idx &= 1; *qual |= CTF_CONST; /* Valarray elements are constant. */ } *pp = p + idx*(int32_t)sz; return ct; } } else if (tviscdata(key)) { /* Integer cdata key. */ GCcdata *cdk = cdataV(key); CType *ctk = ctype_raw(cts, cdk->ctypeid); if (ctype_isenum(ctk->info)) ctk = ctype_child(cts, ctk); if (ctype_isinteger(ctk->info)) { lj_cconv_ct_ct(cts, ctype_get(cts, CTID_INT_PSZ), ctk, (uint8_t *)&idx, cdataptr(cdk), 0); goto integer_key; } } else if (tvisstr(key)) { /* String key. */ GCstr *name = strV(key); if (ctype_isstruct(ct->info)) { CTSize ofs; CType *fct = lj_ctype_getfieldq(cts, ct, name, &ofs, qual); if (fct) { *pp = p + ofs; return fct; } } else if (ctype_iscomplex(ct->info)) { if (name->len == 2) { *qual |= CTF_CONST; /* Complex fields are constant. */ if (strdata(name)[0] == 'r' && strdata(name)[1] == 'e') { *pp = p; return ct; } else if (strdata(name)[0] == 'i' && strdata(name)[1] == 'm') { *pp = p + (ct->size >> 1); return ct; } } } else if (cd->ctypeid == CTID_CTYPEID) {
static void lookahead (LexState *ls) { lua_assert(ls->lookahead.token == TK_EOS); ls->lookahead.token = luaX_lex(ls, &ls->lookahead.seminfo); }
static void propagatelist (global_State *g, GCObject *l) { lua_assert(g->gray == NULL); /* no grays left */ g->gray = l; propagateall(g); /* traverse all elements from 'l' */ }
static void discharge2reg (FuncState *fs, expdesc *e, int reg) { luaK_dischargevars(fs, e); switch (e->k) { case VNIL: { luaK_nil(fs, reg, 1); break; } case VFALSE: case VTRUE: { luaK_codeABC(fs, OP_LOADBOOL, reg, e->k == VTRUE, 0); break; } case VK: { luaK_codek(fs, reg, e->u.info); break; } case VKFLT: { luaK_codek(fs, reg, luaK_numberK(fs, e->u.nval)); break; } case VKINT: { luaK_codek(fs, reg, luaK_intK(fs, e->u.ival)); break; } case VRELOCABLE: { Instruction *pc = &getcode(fs, e); SETARG_A(*pc, reg); DEBUG_EXPR(raviY_printf(fs, "discharge2reg (VRELOCABLE set arg A) %e\n", e)); DEBUG_CODEGEN(raviY_printf(fs, "[%d]* %o ; set A to %d\n", e->u.info, *pc, reg)); break; } case VNONRELOC: { if (reg != e->u.info) { /* code a MOVEI or MOVEF if the target register is a local typed variable */ int ravi_type = raviY_get_register_typeinfo(fs, reg); switch (ravi_type) { case RAVI_TNUMINT: luaK_codeABC(fs, OP_RAVI_MOVEI, reg, e->u.info, 0); break; case RAVI_TNUMFLT: luaK_codeABC(fs, OP_RAVI_MOVEF, reg, e->u.info, 0); break; case RAVI_TARRAYINT: luaK_codeABC(fs, OP_RAVI_MOVEAI, reg, e->u.info, 0); break; case RAVI_TARRAYFLT: luaK_codeABC(fs, OP_RAVI_MOVEAF, reg, e->u.info, 0); break; case RAVI_TTABLE: luaK_codeABC(fs, OP_RAVI_MOVETAB, reg, e->u.info, 0); break; default: luaK_codeABC(fs, OP_MOVE, reg, e->u.info, 0); break; } } break; } default: { lua_assert(e->k == VVOID || e->k == VJMP); return; /* nothing to do... */ } } e->u.info = reg; e->k = VNONRELOC; }
static void freereg (FuncState *fs, int reg) { if (reg >= fs->nactvar && reg < MAXSTACK) { fs->freereg--; lua_assert(reg == fs->freereg); } }
int luaK_codeABx (FuncState *fs, OpCode o, int a, unsigned int bc) { lua_assert(getOpMode(o) == iABx || getOpMode(o) == iAsBx); lua_assert(getCMode(o) == OpArgN); lua_assert(a <= MAXARG_A && bc <= MAXARG_Bx); return luaK_code(fs, CREATE_ABx(o, a, bc)); }
void luaS_freeall (lua_State *L) { lua_assert(G(L)->strt.nuse==0); luaM_freearray(L, G(L)->strt.hash, G(L)->strt.size, TString *); }
static void freereg (FuncState *fs, int reg) { if (!ISK(reg)) { fs->freereg--; lua_assert(reg == fs->freereg); } }
/* Unroll loop. */ static void loop_unroll(jit_State *J) { IRRef1 phi[LJ_MAX_PHI]; uint32_t nphi = 0; IRRef1 *subst; SnapNo onsnap; SnapShot *osnap, *loopsnap; SnapEntry *loopmap, *psentinel; IRRef ins, invar; /* Use temp buffer for substitution table. ** Only non-constant refs in [REF_BIAS,invar) are valid indexes. ** Caveat: don't call into the VM or run the GC or the buffer may be gone. */ invar = J->cur.nins; subst = (IRRef1 *)lj_str_needbuf(J->L, &G(J->L)->tmpbuf, (invar-REF_BIAS)*sizeof(IRRef1)) - REF_BIAS; subst[REF_BASE] = REF_BASE; /* LOOP separates the pre-roll from the loop body. */ emitir_raw(IRTG(IR_LOOP, IRT_NIL), 0, 0); /* Grow snapshot buffer and map for copy-substituted snapshots. ** Need up to twice the number of snapshots minus #0 and loop snapshot. ** Need up to twice the number of entries plus fallback substitutions ** from the loop snapshot entries for each new snapshot. ** Caveat: both calls may reallocate J->cur.snap and J->cur.snapmap! */ onsnap = J->cur.nsnap; lj_snap_grow_buf(J, 2*onsnap-2); lj_snap_grow_map(J, J->cur.nsnapmap*2+(onsnap-2)*J->cur.snap[onsnap-1].nent); /* The loop snapshot is used for fallback substitutions. */ loopsnap = &J->cur.snap[onsnap-1]; loopmap = &J->cur.snapmap[loopsnap->mapofs]; /* The PC of snapshot #0 and the loop snapshot must match. */ psentinel = &loopmap[loopsnap->nent]; lua_assert(*psentinel == J->cur.snapmap[J->cur.snap[0].nent]); *psentinel = SNAP(255, 0, 0); /* Replace PC with temporary sentinel. */ /* Start substitution with snapshot #1 (#0 is empty for root traces). */ osnap = &J->cur.snap[1]; /* Copy and substitute all recorded instructions and snapshots. */ for (ins = REF_FIRST; ins < invar; ins++) { IRIns *ir; IRRef op1, op2; if (ins >= osnap->ref) /* Instruction belongs to next snapshot? */ loop_subst_snap(J, osnap++, loopmap, subst); /* Copy-substitute it. */ /* Substitute instruction operands. */ ir = IR(ins); op1 = ir->op1; if (!irref_isk(op1)) op1 = subst[op1]; op2 = ir->op2; if (!irref_isk(op2)) op2 = subst[op2]; if (irm_kind(lj_ir_mode[ir->o]) == IRM_N && op1 == ir->op1 && op2 == ir->op2) { /* Regular invariant ins? */ subst[ins] = (IRRef1)ins; /* Shortcut. */ } else { /* Re-emit substituted instruction to the FOLD/CSE/etc. pipeline. */ IRType1 t = ir->t; /* Get this first, since emitir may invalidate ir. */ IRRef ref = tref_ref(emitir(ir->ot & ~IRT_ISPHI, op1, op2)); subst[ins] = (IRRef1)ref; if (ref != ins) { IRIns *irr = IR(ref); if (ref < invar) { /* Loop-carried dependency? */ /* Potential PHI? */ if (!irref_isk(ref) && !irt_isphi(irr->t) && !irt_ispri(irr->t)) { irt_setphi(irr->t); if (nphi >= LJ_MAX_PHI) lj_trace_err(J, LJ_TRERR_PHIOV); phi[nphi++] = (IRRef1)ref; } /* Check all loop-carried dependencies for type instability. */ if (!irt_sametype(t, irr->t)) { if (irt_isinteger(t) && irt_isinteger(irr->t)) continue; else if (irt_isnum(t) && irt_isinteger(irr->t)) /* Fix int->num. */ ref = tref_ref(emitir(IRTN(IR_CONV), ref, IRCONV_NUM_INT)); else if (irt_isnum(irr->t) && irt_isinteger(t)) /* Fix num->int. */ ref = tref_ref(emitir(IRTGI(IR_CONV), ref, IRCONV_INT_NUM|IRCONV_CHECK)); else lj_trace_err(J, LJ_TRERR_TYPEINS); subst[ins] = (IRRef1)ref; irr = IR(ref); goto phiconv; } } else if (ref != REF_DROP && irr->o == IR_CONV && ref > invar && irr->op1 < invar) { /* May need an extra PHI for a CONV. */ ref = irr->op1; irr = IR(ref); phiconv: if (ref < invar && !irref_isk(ref) && !irt_isphi(irr->t)) { irt_setphi(irr->t); if (nphi >= LJ_MAX_PHI) lj_trace_err(J, LJ_TRERR_PHIOV); phi[nphi++] = (IRRef1)ref; } } } } } if (!irt_isguard(J->guardemit)) /* Drop redundant snapshot. */ J->cur.nsnapmap = (uint16_t)J->cur.snap[--J->cur.nsnap].mapofs; lua_assert(J->cur.nsnapmap <= J->sizesnapmap); *psentinel = J->cur.snapmap[J->cur.snap[0].nent]; /* Restore PC. */ loop_emit_phi(J, subst, phi, nphi, onsnap); }
/* ** Both NaN and -0.0 should not go to the constant table, as they have ** problems with the hashing. (NaN is not a valid key, -0.0 collides ** with +0.0.) */ static int luaK_numberK (FuncState *fs, lua_Number r) { TValue o; lua_assert(!luai_numisnan(r) && !isminuszero(r)); setfltvalue(&o, r); return addk(fs, &o, &o); }
/* Convert C type to C type. Caveat: expects to get the raw CType! ** ** Note: This is only used by the interpreter and not optimized at all. ** The JIT compiler will do a much better job specializing for each case. */ void lj_cconv_ct_ct(CTState *cts, CType *d, CType *s, uint8_t *dp, uint8_t *sp, CTInfo flags) { CTSize dsize = d->size, ssize = s->size; CTInfo dinfo = d->info, sinfo = s->info; void *tmpptr; lua_assert(!ctype_isenum(dinfo) && !ctype_isenum(sinfo)); lua_assert(!ctype_isattrib(dinfo) && !ctype_isattrib(sinfo)); if (ctype_type(dinfo) > CT_MAYCONVERT || ctype_type(sinfo) > CT_MAYCONVERT) goto err_conv; /* Some basic sanity checks. */ lua_assert(!ctype_isnum(dinfo) || dsize > 0); lua_assert(!ctype_isnum(sinfo) || ssize > 0); lua_assert(!ctype_isbool(dinfo) || dsize == 1 || dsize == 4); lua_assert(!ctype_isbool(sinfo) || ssize == 1 || ssize == 4); lua_assert(!ctype_isinteger(dinfo) || (1u<<lj_fls(dsize)) == dsize); lua_assert(!ctype_isinteger(sinfo) || (1u<<lj_fls(ssize)) == ssize); switch (cconv_idx2(dinfo, sinfo)) { /* Destination is a bool. */ case CCX(B, B): /* Source operand is already normalized. */ if (dsize == 1) *dp = *sp; else *(int *)dp = *sp; break; case CCX(B, I): { MSize i; uint8_t b = 0; for (i = 0; i < ssize; i++) b |= sp[i]; b = (b != 0); if (dsize == 1) *dp = b; else *(int *)dp = b; break; } case CCX(B, F): { uint8_t b; if (ssize == sizeof(double)) b = (*(double *)sp != 0); else if (ssize == sizeof(float)) b = (*(float *)sp != 0); else goto err_conv; /* NYI: long double. */ if (dsize == 1) *dp = b; else *(int *)dp = b; break; } /* Destination is an integer. */ case CCX(I, B): case CCX(I, I): conv_I_I: if (dsize > ssize) { /* Zero-extend or sign-extend LSB. */ #if LJ_LE uint8_t fill = (!(sinfo & CTF_UNSIGNED) && (sp[ssize-1]&0x80)) ? 0xff : 0; memcpy(dp, sp, ssize); memset(dp + ssize, fill, dsize-ssize); #else uint8_t fill = (!(sinfo & CTF_UNSIGNED) && (sp[0]&0x80)) ? 0xff : 0; memset(dp, fill, dsize-ssize); memcpy(dp + (dsize-ssize), sp, ssize); #endif } else { /* Copy LSB. */ #if LJ_LE memcpy(dp, sp, dsize); #else memcpy(dp, sp + (ssize-dsize), dsize); #endif } break; case CCX(I, F): { double n; /* Always convert via double. */ conv_I_F: /* Convert source to double. */ if (ssize == sizeof(double)) n = *(double *)sp; else if (ssize == sizeof(float)) n = (double)*(float *)sp; else goto err_conv; /* NYI: long double. */ /* Then convert double to integer. */ /* The conversion must exactly match the semantics of JIT-compiled code! */ if (dsize < 4 || (dsize == 4 && !(dinfo & CTF_UNSIGNED))) { int32_t i = (int32_t)n; if (dsize == 4) *(int32_t *)dp = i; else if (dsize == 2) *(int16_t *)dp = (int16_t)i; else *(int8_t *)dp = (int8_t)i; } else if (dsize == 4) { *(uint32_t *)dp = (uint32_t)n; } else if (dsize == 8) { if (!(dinfo & CTF_UNSIGNED)) *(int64_t *)dp = (int64_t)n; else *(uint64_t *)dp = lj_num2u64(n); } else { goto err_conv; /* NYI: conversion to >64 bit integers. */ } break; } case CCX(I, C): s = ctype_child(cts, s); sinfo = s->info; ssize = s->size; goto conv_I_F; /* Just convert re. */ case CCX(I, P): if (!(flags & CCF_CAST)) goto err_conv; sinfo = CTINFO(CT_NUM, CTF_UNSIGNED); goto conv_I_I; case CCX(I, A): if (!(flags & CCF_CAST)) goto err_conv; sinfo = CTINFO(CT_NUM, CTF_UNSIGNED); ssize = CTSIZE_PTR; tmpptr = sp; sp = (uint8_t *)&tmpptr; goto conv_I_I; /* Destination is a floating-point number. */ case CCX(F, B): case CCX(F, I): { double n; /* Always convert via double. */ conv_F_I: /* First convert source to double. */ /* The conversion must exactly match the semantics of JIT-compiled code! */ if (ssize < 4 || (ssize == 4 && !(sinfo & CTF_UNSIGNED))) { int32_t i; if (ssize == 4) { i = *(int32_t *)sp; } else if (!(sinfo & CTF_UNSIGNED)) { if (ssize == 2) i = *(int16_t *)sp; else i = *(int8_t *)sp; } else { if (ssize == 2) i = *(uint16_t *)sp; else i = *(uint8_t *)sp; } n = (double)i; } else if (ssize == 4) { n = (double)*(uint32_t *)sp; } else if (ssize == 8) { if (!(sinfo & CTF_UNSIGNED)) n = (double)*(int64_t *)sp; else n = (double)*(uint64_t *)sp; } else { goto err_conv; /* NYI: conversion from >64 bit integers. */ } /* Convert double to destination. */ if (dsize == sizeof(double)) *(double *)dp = n; else if (dsize == sizeof(float)) *(float *)dp = (float)n; else goto err_conv; /* NYI: long double. */ break; } case CCX(F, F): { double n; /* Always convert via double. */ conv_F_F: if (ssize == dsize) goto copyval; /* Convert source to double. */ if (ssize == sizeof(double)) n = *(double *)sp; else if (ssize == sizeof(float)) n = (double)*(float *)sp; else goto err_conv; /* NYI: long double. */ /* Convert double to destination. */ if (dsize == sizeof(double)) *(double *)dp = n; else if (dsize == sizeof(float)) *(float *)dp = (float)n; else goto err_conv; /* NYI: long double. */ break; } case CCX(F, C): s = ctype_child(cts, s); sinfo = s->info; ssize = s->size; goto conv_F_F; /* Ignore im, and convert from re. */ /* Destination is a complex number. */ case CCX(C, I): d = ctype_child(cts, d); dinfo = d->info; dsize = d->size; memset(dp + dsize, 0, dsize); /* Clear im. */ goto conv_F_I; /* Convert to re. */ case CCX(C, F): d = ctype_child(cts, d); dinfo = d->info; dsize = d->size; memset(dp + dsize, 0, dsize); /* Clear im. */ goto conv_F_F; /* Convert to re. */ case CCX(C, C): if (dsize != ssize) { /* Different types: convert re/im separately. */ CType *dc = ctype_child(cts, d); CType *sc = ctype_child(cts, s); lj_cconv_ct_ct(cts, dc, sc, dp, sp, flags); lj_cconv_ct_ct(cts, dc, sc, dp + dc->size, sp + sc->size, flags); return; } goto copyval; /* Otherwise this is easy. */ /* Destination is a vector. */ case CCX(V, I): case CCX(V, F): case CCX(V, C): { CType *dc = ctype_child(cts, d); CTSize esize; /* First convert the scalar to the first element. */ lj_cconv_ct_ct(cts, dc, s, dp, sp, flags); /* Then replicate it to the other elements (splat). */ for (sp = dp, esize = dc->size; dsize > esize; dsize -= esize) { dp += esize; memcpy(dp, sp, esize); } break; } case CCX(V, V): /* Copy same-sized vectors, even for different lengths/element-types. */ if (dsize != ssize) goto err_conv; goto copyval; /* Destination is a pointer. */ case CCX(P, I): if (!(flags & CCF_CAST)) goto err_conv; dinfo = CTINFO(CT_NUM, CTF_UNSIGNED); goto conv_I_I; case CCX(P, F): if (!(flags & CCF_CAST) || !(flags & CCF_FROMTV)) goto err_conv; /* The signed conversion is cheaper. x64 really has 47 bit pointers. */ dinfo = CTINFO(CT_NUM, (LJ_64 && dsize == 8) ? 0 : CTF_UNSIGNED); goto conv_I_F; case CCX(P, P): if (!lj_cconv_compatptr(cts, d, s, flags)) goto err_conv; cdata_setptr(dp, dsize, cdata_getptr(sp, ssize)); break; case CCX(P, A): case CCX(P, S): if (!lj_cconv_compatptr(cts, d, s, flags)) goto err_conv; cdata_setptr(dp, dsize, sp); break; /* Destination is an array. */ case CCX(A, A): if ((flags & CCF_CAST) || (d->info & CTF_VLA) || dsize != ssize || d->size == CTSIZE_INVALID || !lj_cconv_compatptr(cts, d, s, flags)) goto err_conv; goto copyval; /* Destination is a struct/union. */ case CCX(S, S): if ((flags & CCF_CAST) || (d->info & CTF_VLA) || d != s) goto err_conv; /* Must be exact same type. */ copyval: /* Copy value. */ lua_assert(dsize == ssize); memcpy(dp, sp, dsize); break; default: err_conv: cconv_err_conv(cts, d, s, flags); } }
static void invertjump (FuncState *fs, expdesc *e) { Instruction *pc = getjumpcontrol(fs, e->u.info); lua_assert(testTMode(GET_OPCODE(*pc)) && GET_OPCODE(*pc) != OP_TEST); SETARG_A(*pc, !(GETARG_A(*pc))); }
void luaV_execute (lua_State *L, int nexeccalls) { LClosure *cl; StkId base; TValue *k; const Instruction *pc; reentry: /* entry point */ lua_assert(isLua(L->ci)); pc = L->savedpc; cl = &clvalue(L->ci->func)->l; base = L->base; k = cl->p->k; /* main loop of interpreter */ for (;;) { const Instruction i = *pc++; StkId ra; if ((L->hookmask & (LUA_MASKLINE | LUA_MASKCOUNT)) && (--L->hookcount == 0 || L->hookmask & LUA_MASKLINE)) { traceexec(L, pc); if (L->status == LUA_YIELD) { /* did hook yield? */ L->savedpc = pc - 1; return; } base = L->base; } /* warning!! several calls may realloc the stack and invalidate `ra' */ ra = RA(i); lua_assert(base == L->base && L->base == L->ci->base); lua_assert(base <= L->top && L->top <= L->stack + L->stacksize); lua_assert(L->top == L->ci->top || luaG_checkopenop(i)); switch (GET_OPCODE(i)) { case OP_MOVE: { setobjs2s(L, ra, RB(i)); continue; } case OP_LOADK: { setobj2s(L, ra, KBx(i)); continue; } case OP_LOADBOOL: { setbvalue(ra, GETARG_B(i)); if (GETARG_C(i)) pc++; /* skip next instruction (if C) */ continue; } case OP_LOADNIL: { TValue *rb = RB(i); do { setnilvalue(rb--); } while (rb >= ra); continue; } case OP_GETUPVAL: { int b = GETARG_B(i); setobj2s(L, ra, cl->upvals[b]->v); continue; } case OP_GETGLOBAL: { TValue g; TValue *rb = KBx(i); sethvalue(L, &g, cl->env); lua_assert(ttisstring(rb)); Protect(luaV_gettable(L, &g, rb, ra)); continue; } case OP_GETTABLE: { Protect(luaV_gettable(L, RB(i), RKC(i), ra)); continue; } case OP_SETGLOBAL: { TValue g; sethvalue(L, &g, cl->env); lua_assert(ttisstring(KBx(i))); Protect(luaV_settable(L, &g, KBx(i), ra)); continue; } case OP_SETUPVAL: { UpVal *uv = cl->upvals[GETARG_B(i)]; setobj(L, uv->v, ra); luaC_barrier(L, uv, ra); continue; } case OP_SETTABLE: { Protect(luaV_settable(L, ra, RKB(i), RKC(i))); continue; } case OP_NEWTABLE: { int b = GETARG_B(i); int c = GETARG_C(i); sethvalue(L, ra, luaH_new(L, luaO_fb2int(b), luaO_fb2int(c))); Protect(luaC_checkGC(L)); continue; } case OP_SELF: { StkId rb = RB(i); setobjs2s(L, ra+1, rb); Protect(luaV_gettable(L, rb, RKC(i), ra)); continue; } case OP_ADD: { arith_op(luai_numadd, TM_ADD); continue; } case OP_SUB: { arith_op(luai_numsub, TM_SUB); continue; } case OP_MUL: { arith_op(luai_nummul, TM_MUL); continue; } case OP_DIV: { arith_op(luai_lnumdiv, TM_DIV); continue; } case OP_MOD: { arith_op(luai_lnummod, TM_MOD); continue; } case OP_POW: { arith_op(luai_numpow, TM_POW); continue; } case OP_UNM: { TValue *rb = RB(i); if (ttisnumber(rb)) { lua_Number nb = nvalue(rb); setnvalue(ra, luai_numunm(nb)); } else { Protect(Arith(L, ra, rb, rb, TM_UNM)); } continue; } case OP_NOT: { int res = l_isfalse(RB(i)); /* next assignment may change this value */ setbvalue(ra, res); continue; } case OP_LEN: { const TValue *rb = RB(i); switch (ttype(rb)) { case LUA_TTABLE: { setnvalue(ra, cast_num(luaH_getn(hvalue(rb)))); break; } case LUA_TSTRING: { setnvalue(ra, cast_num(tsvalue(rb)->len)); break; } default: { /* try metamethod */ Protect( if (!call_binTM(L, rb, luaO_nilobject, ra, TM_LEN)) luaG_typeerror(L, rb, "get length of"); ) } } continue; } case OP_CONCAT: { int b = GETARG_B(i); int c = GETARG_C(i); Protect(luaV_concat(L, c-b+1, c); luaC_checkGC(L)); setobjs2s(L, RA(i), base+b); continue; } case OP_JMP: { dojump(L, pc, GETARG_sBx(i)); continue; } case OP_EQ: { TValue *rb = RKB(i); TValue *rc = RKC(i); Protect( if (equalobj(L, rb, rc) == GETARG_A(i)) dojump(L, pc, GETARG_sBx(*pc)); ) pc++; continue; } case OP_LT: { Protect( if (luaV_lessthan(L, RKB(i), RKC(i)) == GETARG_A(i)) dojump(L, pc, GETARG_sBx(*pc)); ) pc++; continue; } case OP_LE: { Protect( if (lessequal(L, RKB(i), RKC(i)) == GETARG_A(i)) dojump(L, pc, GETARG_sBx(*pc)); ) pc++; continue; }
/* Array and hash load forwarding. */ static TRef fwd_ahload(jit_State *J, IRRef xref) { IRIns *xr = IR(xref); IRRef lim = xref; /* Search limit. */ IRRef ref; /* Search for conflicting stores. */ ref = J->chain[fins->o+IRDELTA_L2S]; while (ref > xref) { IRIns *store = IR(ref); switch (aa_ahref(J, xr, IR(store->op1))) { case ALIAS_NO: break; /* Continue searching. */ case ALIAS_MAY: lim = ref; goto cselim; /* Limit search for load. */ case ALIAS_MUST: return store->op2; /* Store forwarding. */ } ref = store->prev; } /* No conflicting store (yet): const-fold loads from allocations. */ { IRIns *ir = (xr->o == IR_HREFK || xr->o == IR_AREF) ? IR(xr->op1) : xr; IRRef tab = ir->op1; ir = IR(tab); if (ir->o == IR_TNEW || (ir->o == IR_TDUP && irref_isk(xr->op2))) { /* A NEWREF with a number key may end up pointing to the array part. ** But it's referenced from HSTORE and not found in the ASTORE chain. ** For now simply consider this a conflict without forwarding anything. */ if (xr->o == IR_AREF) { IRRef ref2 = J->chain[IR_NEWREF]; while (ref2 > tab) { IRIns *newref = IR(ref2); if (irt_isnum(IR(newref->op2)->t)) goto cselim; ref2 = newref->prev; } } /* NEWREF inhibits CSE for HREF, and dependent FLOADs from HREFK/AREF. ** But the above search for conflicting stores was limited by xref. ** So continue searching, limited by the TNEW/TDUP. Store forwarding ** is ok, too. A conflict does NOT limit the search for a matching load. */ while (ref > tab) { IRIns *store = IR(ref); switch (aa_ahref(J, xr, IR(store->op1))) { case ALIAS_NO: break; /* Continue searching. */ case ALIAS_MAY: goto cselim; /* Conflicting store. */ case ALIAS_MUST: return store->op2; /* Store forwarding. */ } ref = store->prev; } lua_assert(ir->o != IR_TNEW || irt_isnil(fins->t)); if (irt_ispri(fins->t)) { return TREF_PRI(irt_type(fins->t)); } else if (irt_isnum(fins->t) || irt_isstr(fins->t)) { TValue keyv; cTValue *tv; IRIns *key = IR(xr->op2); if (key->o == IR_KSLOT) key = IR(key->op1); lj_ir_kvalue(J->L, &keyv, key); tv = lj_tab_get(J->L, ir_ktab(IR(ir->op1)), &keyv); lua_assert(itype2irt(tv) == irt_type(fins->t)); if (irt_isnum(fins->t)) return lj_ir_knum_u64(J, tv->u64); else return lj_ir_kstr(J, strV(tv)); } /* Othwerwise: don't intern as a constant. */ } } cselim: /* Try to find a matching load. Below the conflicting store, if any. */ ref = J->chain[fins->o]; while (ref > lim) { IRIns *load = IR(ref); if (load->op1 == xref) return ref; /* Load forwarding. */ ref = load->prev; } return 0; /* Conflict or no match. */ }