/* Helper for FORI. Coercion. */ void LJ_FASTCALL lj_meta_for(lua_State *L, TValue *o) { if (!lj_strscan_numberobj(o)) lj_err_msg(L, LJ_ERR_FORINIT); if (!lj_strscan_numberobj(o+1)) lj_err_msg(L, LJ_ERR_FORLIM); if (!lj_strscan_numberobj(o+2)) lj_err_msg(L, LJ_ERR_FORSTEP); if (LJ_DUALNUM) { /* Ensure all slots are integers or all slots are numbers. */ int32_t k[3]; int nint = 0; ptrdiff_t i; for (i = 0; i <= 2; i++) { if (tvisint(o+i)) { k[i] = intV(o+i); nint++; } else { k[i] = lj_num2int(numV(o+i)); nint += ((lua_Number)k[i] == numV(o+i)); } } if (nint == 3) { /* Narrow to integers. */ setintV(o, k[0]); setintV(o+1, k[1]); setintV(o+2, k[2]); } else if (nint != 0) { /* Widen to numbers. */ if (tvisint(o)) setnumV(o, (lua_Number)intV(o)); if (tvisint(o+1)) setnumV(o+1, (lua_Number)intV(o+1)); if (tvisint(o+2)) setnumV(o+2, (lua_Number)intV(o+2)); } } }
/* Loop optimization. */ int lj_opt_loop(jit_State *J) { IRRef nins = J->cur.nins; MSize nsnap = J->cur.nsnap; int errcode = lj_vm_cpcall(J->L, NULL, J, cploop_opt); if (LJ_UNLIKELY(errcode)) { lua_State *L = J->L; if (errcode == LUA_ERRRUN && tvisnum(L->top-1)) { /* Trace error? */ int32_t e = lj_num2int(numV(L->top-1)); switch ((TraceError)e) { case LJ_TRERR_TYPEINS: /* Type instability. */ case LJ_TRERR_GFAIL: /* Guard would always fail. */ /* Unrolling via recording fixes many cases, e.g. a flipped boolean. */ if (--J->instunroll < 0) /* But do not unroll forever. */ break; L->top--; /* Remove error object. */ loop_undo(J, nins, nsnap); return 1; /* Loop optimization failed, continue recording. */ default: break; } } lj_err_throw(L, errcode); /* Propagate all other errors. */ } return 0; /* Loop optimization is ok. */ }
int32_t lj_lib_checkint(lua_State *L, int narg) { TValue *o = L->base + narg-1; if (!(o < L->top && lj_strscan_numberobj(o))) lj_err_argt(L, narg, LUA_TNUMBER); if (LJ_LIKELY(tvisint(o))) { return intV(o); } else { int32_t i = lj_num2int(numV(o)); if (LJ_DUALNUM) setintV(o, i); return i; } }
/* Record rounding functions math.floor and math.ceil. */ static void LJ_FASTCALL recff_math_round(jit_State *J, RecordFFData *rd) { TRef tr = J->base[0]; if (!tref_isinteger(tr)) { /* Pass through integers unmodified. */ tr = emitir(IRTN(IR_FPMATH), lj_ir_tonum(J, tr), rd->data); /* Result is integral (or NaN/Inf), but may not fit an int32_t. */ if (LJ_DUALNUM) { /* Try to narrow using a guarded conversion to int. */ lua_Number n = lj_vm_foldfpm(numberVnum(&rd->argv[0]), rd->data); if (n == (lua_Number)lj_num2int(n)) tr = emitir(IRTGI(IR_CONV), tr, IRCONV_INT_NUM|IRCONV_CHECK); } J->base[0] = tr; } }
static int io_file_readnum(lua_State *L, FILE *fp) { lua_Number d; if (fscanf(fp, LUA_NUMBER_SCAN, &d) == 1) { if (LJ_DUALNUM) { int32_t i = lj_num2int(d); if (d == (lua_Number)i && !tvismzero((cTValue *)&d)) { setintV(L->top++, i); return 1; } } setnumV(L->top++, d); return 1; } else { setnilV(L->top++); return 0; } }
/* Parse optimization parameter. */ static int jitopt_param(jit_State *J, const char *str) { const char *lst = JIT_P_STRING; int i; for (i = 0; i < JIT_P__MAX; i++) { size_t len = *(const uint8_t *)lst; TValue tv; lua_assert(len != 0); if (strncmp(str, lst+1, len) == 0 && str[len] == '=' && lj_str_numconv(&str[len+1], &tv)) { J->param[i] = lj_num2int(tv.n); if (i == JIT_P_hotloop) lj_dispatch_init_hotcount(J2G(J)); return 1; /* Ok. */ } lst += 1+len; } return 0; /* No match. */ }
/* Parse a number literal. */ static void lex_number(LexState *ls, TValue *tv) { int c; lua_assert(lj_char_isdigit(ls->current)); do { c = ls->current; save_and_next(ls); } while (lj_char_isident(ls->current) || ls->current == '.' || ((ls->current == '-' || ls->current == '+') && ((c & ~0x20) == 'E' || (c & ~0x20) == 'P'))); #if LJ_HASFFI c &= ~0x20; if ((c == 'I' || c == 'L' || c == 'U') && !ctype_ctsG(G(ls->L))) lex_loadffi(ls->L); if (c == 'I') /* Parse imaginary part of complex number. */ ls->sb.n--; #endif save(ls, '\0'); #if LJ_HASFFI if ((c == 'L' || c == 'U') && lex_number64(ls, tv)) { /* Parse 64 bit int. */ return; } else #endif if (lj_str_numconv(ls->sb.buf, tv)) { #if LJ_HASFFI if (c == 'I') { /* Return cdata holding a complex number. */ GCcdata *cd = lj_cdata_new_(ls->L, CTID_COMPLEX_DOUBLE, 2*sizeof(double)); ((double *)cdataptr(cd))[0] = 0; ((double *)cdataptr(cd))[1] = numberVnum(tv); lj_parse_keepcdata(ls, tv, cd); } #endif if (LJ_DUALNUM && tvisnum(tv)) { int32_t k = lj_num2int(numV(tv)); if ((lua_Number)k == numV(tv)) /* -0 cannot end up here. */ setintV(tv, k); } return; } lj_lex_error(ls, TK_number, LJ_ERR_XNUMBER); }
/* Backpropagate narrowing conversion. Return number of needed conversions. */ static int narrow_conv_backprop(NarrowConv *nc, IRRef ref, int depth) { jit_State *J = nc->J; IRIns *ir = IR(ref); IRRef cref; /* Check the easy cases first. */ if (ir->o == IR_CONV && (ir->op2 & IRCONV_SRCMASK) == IRT_INT) { if ((nc->mode & IRCONV_CONVMASK) <= IRCONV_ANY) narrow_stripov_backprop(nc, ir->op1, depth+1); else *nc->sp++ = NARROWINS(NARROW_REF, ir->op1); /* Undo conversion. */ if (nc->t == IRT_I64) *nc->sp++ = NARROWINS(NARROW_SEXT, 0); /* Sign-extend integer. */ return 0; } else if (ir->o == IR_KNUM) { /* Narrow FP constant. */ lua_Number n = ir_knum(ir)->n; if ((nc->mode & IRCONV_CONVMASK) == IRCONV_TOBIT) { /* Allows a wider range of constants. */ int64_t k64 = (int64_t)n; if (n == (lua_Number)k64) { /* Only if const doesn't lose precision. */ *nc->sp++ = NARROWINS(NARROW_INT, 0); *nc->sp++ = (NarrowIns)k64; /* But always truncate to 32 bits. */ return 0; } } else { int32_t k = lj_num2int(n); /* Only if constant is a small integer. */ if (checki16(k) && n == (lua_Number)k) { *nc->sp++ = NARROWINS(NARROW_INT, 0); *nc->sp++ = (NarrowIns)k; return 0; } } return 10; /* Never narrow other FP constants (this is rare). */ } /* Try to CSE the conversion. Stronger checks are ok, too. */ cref = J->chain[fins->o]; while (cref > ref) { IRIns *cr = IR(cref); if (cr->op1 == ref && (fins->o == IR_TOBIT || ((cr->op2 & IRCONV_MODEMASK) == (nc->mode & IRCONV_MODEMASK) && irt_isguard(cr->t) >= irt_isguard(fins->t)))) { *nc->sp++ = NARROWINS(NARROW_REF, cref); return 0; /* Already there, no additional conversion needed. */ } cref = cr->prev; } /* Backpropagate across ADD/SUB. */ if (ir->o == IR_ADD || ir->o == IR_SUB) { /* Try cache lookup first. */ IRRef mode = nc->mode; BPropEntry *bp; /* Inner conversions need a stronger check. */ if ((mode & IRCONV_CONVMASK) == IRCONV_INDEX && depth > 0) mode += IRCONV_CHECK-IRCONV_INDEX; bp = narrow_bpc_get(nc->J, (IRRef1)ref, mode); if (bp) { *nc->sp++ = NARROWINS(NARROW_REF, bp->val); return 0; } else if (nc->t == IRT_I64) { /* Try sign-extending from an existing (checked) conversion to int. */ mode = (IRT_INT<<5)|IRT_NUM|IRCONV_INDEX; bp = narrow_bpc_get(nc->J, (IRRef1)ref, mode); if (bp) { *nc->sp++ = NARROWINS(NARROW_REF, bp->val); *nc->sp++ = NARROWINS(NARROW_SEXT, 0); return 0; } } if (++depth < NARROW_MAX_BACKPROP && nc->sp < nc->maxsp) { NarrowIns *savesp = nc->sp; int count = narrow_conv_backprop(nc, ir->op1, depth); count += narrow_conv_backprop(nc, ir->op2, depth); if (count <= nc->lim) { /* Limit total number of conversions. */ *nc->sp++ = NARROWINS(IRT(ir->o, nc->t), ref); return count; } nc->sp = savesp; /* Too many conversions, need to backtrack. */ } } /* Otherwise add a conversion. */ *nc->sp++ = NARROWINS(NARROW_CONV, ref); return 1; }
/* Get runtime value of int argument. */ static int32_t argv2int(jit_State *J, TValue *o) { if (!tvisnumber(o) && !(tvisstr(o) && lj_str_tonumber(strV(o), o))) lj_trace_err(J, LJ_TRERR_BADTYPE); return tvisint(o) ? intV(o) : lj_num2int(numV(o)); }
/* Index C data by a TValue. Return CType and pointer. */ CType *lj_cdata_index(CTState *cts, GCcdata *cd, cTValue *key, uint8_t **pp, CTInfo *qual) { uint8_t *p = (uint8_t *)cdataptr(cd); CType *ct = ctype_get(cts, cd->ctypeid); ptrdiff_t idx; /* Resolve reference for cdata object. */ if (ctype_isref(ct->info)) { lua_assert(ct->size == CTSIZE_PTR); p = *(uint8_t **)p; ct = ctype_child(cts, ct); } collect_attrib: /* Skip attributes and collect qualifiers. */ while (ctype_isattrib(ct->info)) { if (ctype_attrib(ct->info) == CTA_QUAL) *qual |= ct->size; ct = ctype_child(cts, ct); } lua_assert(!ctype_isref(ct->info)); /* Interning rejects refs to refs. */ if (tvisint(key)) { idx = (ptrdiff_t)intV(key); goto integer_key; } else if (tvisnum(key)) { /* Numeric key. */ #ifdef _MSC_VER /* Workaround for MSVC bug. */ volatile #endif lua_Number n = numV(key); idx = LJ_64 ? (ptrdiff_t)n : (ptrdiff_t)lj_num2int(n); integer_key: if (ctype_ispointer(ct->info)) { CTSize sz = lj_ctype_size(cts, ctype_cid(ct->info)); /* Element size. */ if (sz == CTSIZE_INVALID) lj_err_caller(cts->L, LJ_ERR_FFI_INVSIZE); if (ctype_isptr(ct->info)) { p = (uint8_t *)cdata_getptr(p, ct->size); } else if ((ct->info & (CTF_VECTOR|CTF_COMPLEX))) { if ((ct->info & CTF_COMPLEX)) idx &= 1; *qual |= CTF_CONST; /* Valarray elements are constant. */ } *pp = p + idx*(int32_t)sz; return ct; } } else if (tviscdata(key)) { /* Integer cdata key. */ GCcdata *cdk = cdataV(key); CType *ctk = ctype_raw(cts, cdk->ctypeid); if (ctype_isenum(ctk->info)) ctk = ctype_child(cts, ctk); if (ctype_isinteger(ctk->info)) { lj_cconv_ct_ct(cts, ctype_get(cts, CTID_INT_PSZ), ctk, (uint8_t *)&idx, cdataptr(cdk), 0); goto integer_key; } } else if (tvisstr(key)) { /* String key. */ GCstr *name = strV(key); if (ctype_isstruct(ct->info)) { CTSize ofs; CType *fct = lj_ctype_getfieldq(cts, ct, name, &ofs, qual); if (fct) { *pp = p + ofs; return fct; } } else if (ctype_iscomplex(ct->info)) { if (name->len == 2) { *qual |= CTF_CONST; /* Complex fields are constant. */ if (strdata(name)[0] == 'r' && strdata(name)[1] == 'e') { *pp = p; return ct; } else if (strdata(name)[0] == 'i' && strdata(name)[1] == 'm') { *pp = p + (ct->size >> 1); return ct; } } } else if (cd->ctypeid == CTID_CTYPEID) {
/* Check whether a number fits into an int32_t (-0 is ok, too). */ static int numisint(lua_Number n) { return (n == cast_num(lj_num2int(n))); }
/* Backpropagate narrowing conversion. Return number of needed conversions. */ static int narrow_conv_backprop(NarrowConv *nc, IRRef ref, int depth) { jit_State *J = nc->J; IRIns *ir = IR(ref); IRRef cref; /* Check the easy cases first. */ if (ir->o == IR_TONUM) { /* Undo inverse conversion. */ *nc->sp++ = NARROWINS(NARROW_REF, ir->op1); return 0; } else if (ir->o == IR_KNUM) { /* Narrow FP constant. */ lua_Number n = ir_knum(ir)->n; if (nc->mode == IRTOINT_TOBIT) { /* Allows a wider range of constants. */ int64_t k64 = (int64_t)n; if (n == cast_num(k64)) { /* Only if constant doesn't lose precision. */ *nc->sp++ = NARROWINS(NARROW_INT, 0); *nc->sp++ = (NarrowIns)k64; /* But always truncate to 32 bits. */ return 0; } } else { int32_t k = lj_num2int(n); if (n == cast_num(k)) { /* Only if constant is really an integer. */ *nc->sp++ = NARROWINS(NARROW_INT, 0); *nc->sp++ = (NarrowIns)k; return 0; } } return 10; /* Never narrow other FP constants (this is rare). */ } /* Try to CSE the conversion. Stronger checks are ok, too. */ for (cref = J->chain[fins->o]; cref > ref; cref = IR(cref)->prev) if (IR(cref)->op1 == ref && irt_isguard(IR(cref)->t) >= irt_isguard(fins->t)) { *nc->sp++ = NARROWINS(NARROW_REF, cref); return 0; /* Already there, no additional conversion needed. */ } /* Backpropagate across ADD/SUB. */ if (ir->o == IR_ADD || ir->o == IR_SUB) { /* Try cache lookup first. */ IRRef bpref, mode = nc->mode; if (mode == IRTOINT_INDEX && depth > 0) mode = IRTOINT_CHECK; /* Inner conversions need a stronger check. */ bpref = narrow_bpc_get(nc->J, (IRRef1)ref, mode); if (bpref) { *nc->sp++ = NARROWINS(NARROW_REF, bpref); return 0; } if (++depth < NARROW_MAX_BACKPROP && nc->sp < nc->maxsp) { NarrowIns *savesp = nc->sp; int count = narrow_conv_backprop(nc, ir->op1, depth); count += narrow_conv_backprop(nc, ir->op2, depth); if (count <= nc->lim) { /* Limit total number of conversions. */ *nc->sp++ = NARROWINS(IRTI(ir->o), ref); return count; } nc->sp = savesp; /* Too many conversions, need to backtrack. */ } } /* Otherwise add a conversion. */ *nc->sp++ = NARROWINS(NARROW_CONV, ref); return 1; }
/* Get runtime value of int argument. */ static int32_t argv2int(jit_State *J, TValue *o) { if (!lj_strscan_numberobj(o)) lj_trace_err(J, LJ_TRERR_BADTYPE); return tvisint(o) ? intV(o) : lj_num2int(numV(o)); }