/* Setup call to metamethod to be run by Assembler VM. */ static TValue *mmcall(lua_State *L, ASMFunction cont, cTValue *mo, cTValue *a, cTValue *b) { /* ** |-- framesize -> top top+1 top+2 top+3 ** before: [func slots ...] ** mm setup: [func slots ...] [cont|?] [mo|tmtype] [a] [b] ** in asm: [func slots ...] [cont|PC] [mo|delta] [a] [b] ** ^-- func base ^-- mm base ** after mm: [func slots ...] [result] ** ^-- copy to base[PC_RA] --/ for lj_cont_ra ** istruecond + branch for lj_cont_cond* ** ignore for lj_cont_nop ** next PC: [func slots ...] */ TValue *top = L->top; if (curr_funcisL(L)) top = curr_topL(L); setcont(top++, cont); /* Assembler VM stores PC in upper word or FR2. */ if (LJ_FR2) setnilV(top++); copyTV(L, top++, mo); /* Store metamethod and two arguments. */ if (LJ_FR2) setnilV(top++); copyTV(L, top, a); copyTV(L, top+1, b); return top; /* Return new base. */ }
/* Helper for ISTYPE and ISNUM. Implicit coercion or error. */ void lj_meta_istype(lua_State *L, BCReg ra, BCReg tp) { L->top = curr_topL(L); ra++; tp--; lua_assert(LJ_DUALNUM || tp != ~LJ_TNUMX); /* ISTYPE -> ISNUM broken. */ if (LJ_DUALNUM && tp == ~LJ_TNUMX) lj_lib_checkint(L, ra); else if (tp == ~LJ_TNUMX+1) lj_lib_checknum(L, ra); else if (tp == ~LJ_TSTR) lj_lib_checkstr(L, ra); else lj_err_argtype(L, ra, lj_obj_itypename[tp]); }
/* Formatted runtime error message. */ LJ_NORET LJ_NOINLINE static void err_msgv(lua_State *L, ErrMsg em, ...) { const char *msg; va_list argp; va_start(argp, em); if (curr_funcisL(L)) L->top = curr_topL(L); msg = lj_str_pushvf(L, err2msg(em), argp); va_end(argp); lj_debug_addloc(L, msg, L->base-1, NULL); lj_err_run(L); }
/* Restore interpreter state from exit state with the help of a snapshot. */ const BCIns *lj_snap_restore(jit_State *J, void *exptr) { ExitState *ex = (ExitState *)exptr; SnapNo snapno = J->exitno; /* For now, snapno == exitno. */ GCtrace *T = traceref(J, J->parent); SnapShot *snap = &T->snap[snapno]; MSize n, nent = snap->nent; SnapEntry *map = &T->snapmap[snap->mapofs]; SnapEntry *flinks = &T->snapmap[snap_nextofs(T, snap)-1]; ptrdiff_t ftsz0; TValue *frame; BloomFilter rfilt = snap_renamefilter(T, snapno); const BCIns *pc = snap_pc(map[nent]); lua_State *L = J->L; /* Set interpreter PC to the next PC to get correct error messages. */ setcframe_pc(cframe_raw(L->cframe), pc+1); /* Make sure the stack is big enough for the slots from the snapshot. */ if (LJ_UNLIKELY(L->base + snap->topslot >= tvref(L->maxstack))) { L->top = curr_topL(L); lj_state_growstack(L, snap->topslot - curr_proto(L)->framesize); } /* Fill stack slots with data from the registers and spill slots. */ frame = L->base-1; ftsz0 = frame_ftsz(frame); /* Preserve link to previous frame in slot #0. */ for (n = 0; n < nent; n++) { SnapEntry sn = map[n]; if (!(sn & SNAP_NORESTORE)) { TValue *o = &frame[snap_slot(sn)]; IRRef ref = snap_ref(sn); IRIns *ir = &T->ir[ref]; if (ir->r == RID_SUNK) { MSize j; for (j = 0; j < n; j++) if (snap_ref(map[j]) == ref) { /* De-duplicate sunk allocations. */ copyTV(L, o, &frame[snap_slot(map[j])]); goto dupslot; } snap_unsink(J, T, ex, snapno, rfilt, ir, o); dupslot: continue; } snap_restoreval(J, T, ex, snapno, rfilt, ref, o); if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM) && tvisint(o)) { TValue tmp; snap_restoreval(J, T, ex, snapno, rfilt, ref+1, &tmp); o->u32.hi = tmp.u32.lo; } else if ((sn & (SNAP_CONT|SNAP_FRAME))) { lua_assert(!LJ_FR2); /* TODO_FR2: store 64 bit PCs. */ /* Overwrite tag with frame link. */ setframe_ftsz(o, snap_slot(sn) != 0 ? (int32_t)*flinks-- : ftsz0); L->base = o+1; } } } lua_assert(map + nent == flinks); /* Compute current stack top. */ switch (bc_op(*pc)) { default: if (bc_op(*pc) < BC_FUNCF) { L->top = curr_topL(L); break; } /* fallthrough */ case BC_CALLM: case BC_CALLMT: case BC_RETM: case BC_TSETM: L->top = frame + snap->nslots; break; } return pc; }
/* Helper for CAT. Coercion, iterative concat, __concat metamethod. */ TValue *lj_meta_cat(lua_State *L, TValue *top, int left) { int fromc = 0; if (left < 0) { left = -left; fromc = 1; } do { if (!(tvisstr(top) || tvisnumber(top)) || !(tvisstr(top-1) || tvisnumber(top-1))) { cTValue *mo = lj_meta_lookup(L, top-1, MM_concat); if (tvisnil(mo)) { mo = lj_meta_lookup(L, top, MM_concat); if (tvisnil(mo)) { if (tvisstr(top-1) || tvisnumber(top-1)) top++; lj_err_optype(L, top-1, LJ_ERR_OPCAT); return NULL; /* unreachable */ } } /* One of the top two elements is not a string, call __cat metamethod: ** ** before: [...][CAT stack .........................] ** top-1 top top+1 top+2 ** pick two: [...][CAT stack ...] [o1] [o2] ** setup mm: [...][CAT stack ...] [cont|?] [mo|tmtype] [o1] [o2] ** in asm: [...][CAT stack ...] [cont|PC] [mo|delta] [o1] [o2] ** ^-- func base ^-- mm base ** after mm: [...][CAT stack ...] <--push-- [result] ** next step: [...][CAT stack .............] */ copyTV(L, top+2*LJ_FR2+2, top); /* Carefully ordered stack copies! */ copyTV(L, top+2*LJ_FR2+1, top-1); copyTV(L, top+LJ_FR2, mo); setcont(top-1, lj_cont_cat); if (LJ_FR2) { setnilV(top); setnilV(top+2); top += 2; } return top+1; /* Trigger metamethod call. */ } else { /* Pick as many strings as possible from the top and concatenate them: ** ** before: [...][CAT stack ...........................] ** pick str: [...][CAT stack ...] [...... strings ......] ** concat: [...][CAT stack ...] [result] ** next step: [...][CAT stack ............] */ TValue *e, *o = top; uint64_t tlen = tvisstr(o) ? strV(o)->len : STRFMT_MAXBUF_NUM; char *p, *buf; do { o--; tlen += tvisstr(o) ? strV(o)->len : STRFMT_MAXBUF_NUM; } while (--left > 0 && (tvisstr(o-1) || tvisnumber(o-1))); if (tlen >= LJ_MAX_STR) lj_err_msg(L, LJ_ERR_STROV); p = buf = lj_buf_tmp(L, (MSize)tlen); for (e = top, top = o; o <= e; o++) { if (tvisstr(o)) { GCstr *s = strV(o); MSize len = s->len; p = lj_buf_wmem(p, strdata(s), len); } else if (tvisint(o)) { p = lj_strfmt_wint(p, intV(o)); } else { lua_assert(tvisnum(o)); p = lj_strfmt_wnum(p, o); } } setstrV(L, top, lj_str_new(L, buf, (size_t)(p-buf))); } } while (left >= 1); if (LJ_UNLIKELY(G(L)->gc.total >= G(L)->gc.threshold)) { if (!fromc) L->top = curr_topL(L); lj_gc_step(L); } return NULL; }
/* Restore interpreter state from exit state with the help of a snapshot. */ const BCIns *lj_snap_restore(jit_State *J, void *exptr) { ExitState *ex = (ExitState *)exptr; SnapNo snapno = J->exitno; /* For now, snapno == exitno. */ GCtrace *T = traceref(J, J->parent); SnapShot *snap = &T->snap[snapno]; MSize n, nent = snap->nent; SnapEntry *map = &T->snapmap[snap->mapofs]; SnapEntry *flinks = map + nent + snap->depth; int32_t ftsz0; BCReg nslots = snap->nslots; TValue *frame; BloomFilter rfilt = snap_renamefilter(T, snapno); const BCIns *pc = snap_pc(map[nent]); lua_State *L = J->L; /* Set interpreter PC to the next PC to get correct error messages. */ setcframe_pc(cframe_raw(L->cframe), pc+1); /* Make sure the stack is big enough for the slots from the snapshot. */ if (LJ_UNLIKELY(L->base + nslots > tvref(L->maxstack))) { L->top = curr_topL(L); lj_state_growstack(L, nslots - curr_proto(L)->framesize); } /* Fill stack slots with data from the registers and spill slots. */ frame = L->base-1; ftsz0 = frame_ftsz(frame); /* Preserve link to previous frame in slot #0. */ for (n = 0; n < nent; n++) { SnapEntry sn = map[n]; IRRef ref = snap_ref(sn); BCReg s = snap_slot(sn); TValue *o = &frame[s]; /* Stack slots are relative to start frame. */ IRIns *ir = &T->ir[ref]; if (irref_isk(ref)) { /* Restore constant slot. */ lj_ir_kvalue(L, o, ir); if ((sn & (SNAP_CONT|SNAP_FRAME))) { /* Overwrite tag with frame link. */ o->fr.tp.ftsz = s != 0 ? (int32_t)*flinks-- : ftsz0; if ((sn & SNAP_FRAME)) { GCfunc *fn = ir_kfunc(ir); if (isluafunc(fn)) { MSize framesize = funcproto(fn)->framesize; L->base = ++o; if (LJ_UNLIKELY(o + framesize > tvref(L->maxstack))) { ptrdiff_t fsave = savestack(L, frame); L->top = o; lj_state_growstack(L, framesize); /* Grow again. */ frame = restorestack(L, fsave); } } } } } else if (!(sn & SNAP_NORESTORE)) { IRType1 t = ir->t; RegSP rs = ir->prev; lua_assert(!(sn & (SNAP_CONT|SNAP_FRAME))); if (LJ_UNLIKELY(bloomtest(rfilt, ref))) rs = snap_renameref(T, snapno, ref, rs); if (ra_hasspill(regsp_spill(rs))) { /* Restore from spill slot. */ int32_t *sps = &ex->spill[regsp_spill(rs)]; if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) { o->u32.lo = (uint32_t)*sps; } else if (irt_isinteger(t)) { setintV(o, *sps); #if !LJ_SOFTFP } else if (irt_isnum(t)) { o->u64 = *(uint64_t *)sps; #endif #if LJ_64 } else if (irt_islightud(t)) { /* 64 bit lightuserdata which may escape already has the tag bits. */ o->u64 = *(uint64_t *)sps; #endif } else { lua_assert(!irt_ispri(t)); /* PRI refs never have a spill slot. */ setgcrefi(o->gcr, *sps); setitype(o, irt_toitype(t)); } } else { /* Restore from register. */ Reg r = regsp_reg(rs); lua_assert(ra_hasreg(r)); if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) { o->u32.lo = (uint32_t)ex->gpr[r-RID_MIN_GPR]; } else if (irt_isinteger(t)) { setintV(o, (int32_t)ex->gpr[r-RID_MIN_GPR]); #if !LJ_SOFTFP } else if (irt_isnum(t)) { setnumV(o, ex->fpr[r-RID_MIN_FPR]); #endif #if LJ_64 } else if (irt_islightud(t)) { /* 64 bit lightuserdata which may escape already has the tag bits. */ o->u64 = ex->gpr[r-RID_MIN_GPR]; #endif } else { if (!irt_ispri(t)) setgcrefi(o->gcr, ex->gpr[r-RID_MIN_GPR]); setitype(o, irt_toitype(t)); } } if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) { rs = (ir+1)->prev; if (LJ_UNLIKELY(bloomtest(rfilt, ref+1))) rs = snap_renameref(T, snapno, ref+1, rs); o->u32.hi = (ra_hasspill(regsp_spill(rs))) ? (uint32_t)*&ex->spill[regsp_spill(rs)] : (uint32_t)ex->gpr[regsp_reg(rs)-RID_MIN_GPR]; } } } switch (bc_op(*pc)) { case BC_CALLM: case BC_CALLMT: case BC_RETM: case BC_TSETM: L->top = frame + nslots; break; default: L->top = curr_topL(L); break; } lua_assert(map + nent == flinks); return pc; }
/* Restore interpreter state from exit state with the help of a snapshot. */ void lj_snap_restore(jit_State *J, void *exptr) { ExitState *ex = (ExitState *)exptr; SnapNo snapno = J->exitno; /* For now, snapno == exitno. */ Trace *T = J->trace[J->parent]; SnapShot *snap = &T->snap[snapno]; BCReg s, nslots = snap->nslots; IRRef2 *map = &T->snapmap[snap->mapofs]; IRRef2 *flinks = map + nslots + snap->nframelinks; TValue *o, *newbase, *ntop; BloomFilter rfilt = snap_renamefilter(T, snapno); lua_State *L = J->L; /* Make sure the stack is big enough for the slots from the snapshot. */ if (L->base + nslots >= L->maxstack) { L->top = curr_topL(L); lj_state_growstack(L, nslots - curr_proto(L)->framesize); } /* Fill stack slots with data from the registers and spill slots. */ newbase = NULL; ntop = L->base; for (s = 0, o = L->base-1; s < nslots; s++, o++) { IRRef ref = snap_ref(map[s]); if (ref) { IRIns *ir = &T->ir[ref]; if (irref_isk(ref)) { /* Restore constant slot. */ lj_ir_kvalue(L, o, ir); } else { IRType1 t = ir->t; RegSP rs = ir->prev; if (LJ_UNLIKELY(bloomtest(rfilt, ref))) rs = snap_renameref(T, snapno, ref, rs); if (ra_hasspill(regsp_spill(rs))) { /* Restore from spill slot. */ int32_t *sps = &ex->spill[regsp_spill(rs)]; if (irt_isinteger(t)) { setintV(o, *sps); } else if (irt_isnum(t)) { o->u64 = *(uint64_t *)sps; } else { lua_assert(!irt_ispri(t)); /* PRI refs never have a spill slot. */ setgcrefi(o->gcr, *sps); setitype(o, irt_toitype(t)); } } else if (ra_hasreg(regsp_reg(rs))) { /* Restore from register. */ Reg r = regsp_reg(rs); if (irt_isinteger(t)) { setintV(o, ex->gpr[r-RID_MIN_GPR]); } else if (irt_isnum(t)) { setnumV(o, ex->fpr[r-RID_MIN_FPR]); } else { if (!irt_ispri(t)) setgcrefi(o->gcr, ex->gpr[r-RID_MIN_GPR]); setitype(o, irt_toitype(t)); } } else { /* Restore frame slot. */ lua_assert(ir->o == IR_FRAME); /* This works for both PTR and FUNC IR_FRAME. */ setgcrefp(o->fr.func, mref(T->ir[ir->op2].ptr, void)); if (s != 0) /* Do not overwrite link to previous frame. */ o->fr.tp.ftsz = (int32_t)*--flinks; if (irt_isfunc(ir->t)) { GCfunc *fn = gco2func(gcref(T->ir[ir->op2].gcr)); if (isluafunc(fn)) { TValue *fs; newbase = o+1; fs = newbase + funcproto(fn)->framesize; if (fs > ntop) ntop = fs; /* Update top for newly added frames. */ } } } } } else if (newbase) { setnilV(o); /* Clear unreferenced slots of newly added frames. */ } }
/* Helper for CAT. Coercion, iterative concat, __concat metamethod. */ TValue *lj_meta_cat(lua_State *L, TValue *top, int left) { int fromc = 0; if (left < 0) { left = -left; fromc = 1; } do { int n = 1; if (!(tvisstr(top-1) || tvisnumber(top-1)) || !tostring(L, top)) { cTValue *mo = lj_meta_lookup(L, top-1, MM_concat); if (tvisnil(mo)) { mo = lj_meta_lookup(L, top, MM_concat); if (tvisnil(mo)) { if (tvisstr(top-1) || tvisnumber(top-1)) top++; lj_err_optype(L, top-1, LJ_ERR_OPCAT); return NULL; /* unreachable */ } } /* One of the top two elements is not a string, call __cat metamethod: ** ** before: [...][CAT stack .........................] ** top-1 top top+1 top+2 ** pick two: [...][CAT stack ...] [o1] [o2] ** setup mm: [...][CAT stack ...] [cont|?] [mo|tmtype] [o1] [o2] ** in asm: [...][CAT stack ...] [cont|PC] [mo|delta] [o1] [o2] ** ^-- func base ^-- mm base ** after mm: [...][CAT stack ...] <--push-- [result] ** next step: [...][CAT stack .............] */ copyTV(L, top+2, top); /* Careful with the order of stack copies! */ copyTV(L, top+1, top-1); copyTV(L, top, mo); setcont(top-1, lj_cont_cat); return top+1; /* Trigger metamethod call. */ } else if (strV(top)->len == 0) { /* Shortcut. */ (void)tostring(L, top-1); } else { /* Pick as many strings as possible from the top and concatenate them: ** ** before: [...][CAT stack ...........................] ** pick str: [...][CAT stack ...] [...... strings ......] ** concat: [...][CAT stack ...] [result] ** next step: [...][CAT stack ............] */ MSize tlen = strV(top)->len; char *buffer; int i; for (n = 1; n <= left && tostring(L, top-n); n++) { MSize len = strV(top-n)->len; if (len >= LJ_MAX_STR - tlen) lj_err_msg(L, LJ_ERR_STROV); tlen += len; } buffer = lj_str_needbuf(L, &G(L)->tmpbuf, tlen); n--; tlen = 0; for (i = n; i >= 0; i--) { MSize len = strV(top-i)->len; memcpy(buffer + tlen, strVdata(top-i), len); tlen += len; } setstrV(L, top-n, lj_str_new(L, buffer, tlen)); } left -= n; top -= n; } while (left >= 1); if (LJ_UNLIKELY(G(L)->gc.total >= G(L)->gc.threshold)) { if (!fromc) L->top = curr_topL(L); lj_gc_step(L); } return NULL; }