void storeTV(Vout& v, Vptr dst, Vloc srcLoc, const SSATmp* src) {
auto const type = src->type();
if (srcLoc.isFullSIMD()) {
// The whole TV is stored in a single SIMD reg.
assertx(RuntimeOption::EvalHHIRAllocSIMDRegs);
v << storeups{srcLoc.reg(), dst};
return;
}
if (type.needsReg()) {
assertx(srcLoc.hasReg(1));
v << storeb{srcLoc.reg(1), dst + TVOFF(m_type)};
} else {
v << storeb{v.cns(type.toDataType()), dst + TVOFF(m_type)};
}
// We ignore the values of statically nullish types.
if (src->isA(TNull) || src->isA(TNullptr)) return;
// Store the value.
if (src->hasConstVal()) {
// Skip potential zero-extend if we know the value.
v << store{v.cns(src->rawVal()), dst + TVOFF(m_data)};
} else {
assertx(srcLoc.hasReg(0));
auto const extended = zeroExtendIfBool(v, src->type(), srcLoc.reg(0));
v << store{extended, dst + TVOFF(m_data)};
}
}
void loadTV(Vout& v, const SSATmp* dst, Vloc dstLoc, Vptr src,
bool aux /* = false */) {
auto const type = dst->type();
if (dstLoc.isFullSIMD()) {
// The whole TV is loaded into a single SIMD reg.
assertx(RuntimeOption::EvalHHIRAllocSIMDRegs);
v << loadups{src, dstLoc.reg()};
return;
}
if (type.needsReg()) {
assertx(dstLoc.hasReg(1));
if (aux) {
v << load{src + TVOFF(m_type), dstLoc.reg(1)};
} else {
v << loadb{src + TVOFF(m_type), dstLoc.reg(1)};
}
}
if (type <= TBool) {
v << loadtqb{src + TVOFF(m_data), dstLoc.reg(0)};
} else {
v << load{src + TVOFF(m_data), dstLoc.reg(0)};
}
}
void cgCheckRefInner(IRLS& env, const IRInstruction* inst) {
if (inst->typeParam() >= TInitCell) return;
auto const base = srcLoc(env, inst, 0).reg()[RefData::tvOffset()];
emitTypeCheck(vmain(env), env, inst->typeParam(),
base + TVOFF(m_type), base + TVOFF(m_data), inst->taken());
}
void cgCheckStk(IRLS& env, const IRInstruction* inst) {
auto const baseOff = cellsToBytes(inst->extra<CheckStk>()->offset.offset);
auto const base = srcLoc(env, inst, 0).reg()[baseOff];
emitTypeCheck(vmain(env), env, inst->typeParam(),
base + TVOFF(m_type), base + TVOFF(m_data), inst->taken());
}
void cgCheckLoc(IRLS& env, const IRInstruction* inst) {
auto const baseOff = localOffset(inst->extra<CheckLoc>()->locId);
auto const base = srcLoc(env, inst, 0).reg()[baseOff];
emitTypeCheck(vmain(env), env, inst->typeParam(),
base + TVOFF(m_type), base + TVOFF(m_data), inst->taken());
}
void cgLdLocPseudoMain(IRLS& env, const IRInstruction* inst) {
auto const fp = srcLoc(env, inst, 0).reg();
auto const off = localOffset(inst->extra<LdLocPseudoMain>()->locId);
auto& v = vmain(env);
irlower::emitTypeCheck(v, env, inst->typeParam(), fp[off + TVOFF(m_type)],
fp[off + TVOFF(m_data)], inst->taken());
loadTV(v, inst->dst(), dstLoc(env, inst, 0), fp[off]);
}
void cgLdPropAddr(IRLS& env, const IRInstruction* inst) {
auto& v = vmain(env);
auto const dstLoc = irlower::dstLoc(env, inst, 0);
auto const valReg = dstLoc.reg(tv_lval::val_idx);
auto const propPtr =
srcLoc(env, inst, 0).reg()[inst->extra<LdPropAddr>()->offsetBytes];
v << lea{propPtr, valReg};
if (wide_tv_val) {
static_assert(TVOFF(m_data) == 0, "");
v << lea{propPtr + TVOFF(m_type), dstLoc.reg(tv_lval::type_idx)};
}
}
void CodeGenerator::cgGuardLoc(IRInstruction* inst) {
auto const rFP = x2a(m_regs[inst->src(0)].reg());
auto const baseOff = localOffset(inst->extra<GuardLoc>()->locId);
emitTypeTest(
inst->typeParam(),
rFP[baseOff + TVOFF(m_type)],
rFP[baseOff + TVOFF(m_data)],
[&] (ConditionCode cc) {
auto const destSK = SrcKey(curFunc(), m_unit.bcOff());
auto const destSR = m_tx64->getSrcRec(destSK);
destSR->emitFallbackJump(this->m_mainCode, ccNegate(cc));
});
}
void CodeGenerator::cgGuardStk(IRInstruction* inst) {
auto const rSP = x2a(curOpd(inst->src(0)).reg());
auto const baseOff = cellsToBytes(inst->extra<GuardStk>()->offset);
emitTypeTest(
inst->typeParam(),
rSP[baseOff + TVOFF(m_type)],
rSP[baseOff + TVOFF(m_data)],
[&] (ConditionCode cc) {
auto const destSK = SrcKey(curFunc(), m_unit.bcOff());
auto const destSR = m_tx64->getSrcRec(destSK);
destSR->emitFallbackJump(this->m_mainCode, ccNegate(cc));
});
}
void CodeGenerator::cgSideExitGuardStk(IRInstruction* inst) {
auto const sp = x2a(curOpd(inst->src(0)).reg());
auto const extra = inst->extra<SideExitGuardStk>();
emitTypeTest(
inst->typeParam(),
sp[cellsToBytes(extra->checkedSlot) + TVOFF(m_type)],
sp[cellsToBytes(extra->checkedSlot) + TVOFF(m_data)],
[&] (ConditionCode cc) {
auto const sk = SrcKey(curFunc(), extra->taken);
emitBindSideExit(this->m_mainCode, this->m_stubsCode, sk, ccNegate(cc));
}
);
}
void CodeGenerator::cgSpillStack(IRInstruction* inst) {
// TODO(2966414): so much of this logic could be shared. The opcode itself
// should probably be broken up.
SSATmp* dst = inst->dst();
SSATmp* sp = inst->src(0);
auto const spDeficit = inst->src(1)->getValInt();
auto const spillVals = inst->srcs().subpiece(2);
auto const numSpillSrcs = spillVals.size();
auto const dstReg = x2a(curOpd(dst).reg());
auto const spReg = x2a(curOpd(sp).reg());
auto const spillCells = spillValueCells(inst);
int64_t adjustment = (spDeficit - spillCells) * sizeof(Cell);
for (uint32_t i = 0; i < numSpillSrcs; ++i) {
const int64_t offset = i * sizeof(Cell) + adjustment;
auto* val = spillVals[i];
if (val->type() == Type::None) {
// The simplifier detected that this store was redundnant.
continue;
}
// XXX this is a cut-down version of cgStore.
if (val->isConst()) {
m_as. Mov (rAsm, val->getValBits());
m_as. Str (rAsm, spReg[offset]);
} else {
auto reg = x2a(curOpd(val).reg());
m_as. Str (reg, spReg[offset]);
}
m_as. Mov (rAsm, val->type().toDataType());
m_as. Strb (rAsm.W(), spReg[offset + TVOFF(m_type)]);
}
emitRegGetsRegPlusImm(m_as, dstReg, spReg, adjustment);
}
void cgContArIncKey(IRLS& env, const IRInstruction* inst) {
auto const contAR = srcLoc(env, inst, 0).reg();
auto& v = vmain(env);
auto const keyOff = GENDATAOFF(m_key) - Generator::arOff();
v << incqm{contAR[keyOff + TVOFF(m_data)], v.makeReg()};
}
/*
* Helper for the freeLocalsHelpers which does the actual work of decrementing
* a value's refcount or releasing it.
*
* This helper is reached via call from the various freeLocalHelpers. It
* expects `tv' to be the address of a TypedValue with refcounted type `type'
* (though it may be static, and we will do nothing in that case).
*
* The `live' registers must be preserved across any native calls (and
* generally left untouched).
*/
static TCA emitDecRefHelper(CodeBlock& cb, DataBlock& data, CGMeta& fixups,
PhysReg tv, PhysReg type, RegSet live) {
return vwrap(cb, data, fixups, [&] (Vout& v) {
// We use the first argument register for the TV data because we might pass
// it to the native release call. It's not live when we enter the helper.
auto const data = rarg(0);
v << load{tv[TVOFF(m_data)], data};
auto destroy = [&](Vout& v) {
PhysRegSaver prs{v, live};
auto const dword_size = sizeof(int64_t);
// saving return value on the stack, but keeping it 16-byte aligned
v << mflr{rfuncln()};
v << lea {rsp()[-2 * dword_size], rsp()};
v << store{rfuncln(), rsp()[0]};
// The refcount is exactly 1; release the value.
// Avoid 'this' pointer overwriting by reserving it as an argument.
v << callm{lookupDestructor(v, type), arg_regs(1)};
// Between where r1 is now and the saved RIP of the call into the
// freeLocalsHelpers stub, we have all the live regs we pushed, plus the
// stack size reserved for the LR saved right above and the LR offset in
// the frame.
v << syncpoint{makeIndirectFixup(prs.dwordsPushed())};
// fallthru
// restore the return value from the stack
v << load{rsp()[0], rfuncln()};
v << lea {rsp()[2 * dword_size], rsp()};
v << mtlr{rfuncln()};
};
auto const sf = emitCmpRefCount(v, OneReference, data);
if (one_bit_refcount) {
ifThen(v, CC_E, sf, destroy);
} else {
ifThen(v, CC_NL, sf, [&] (Vout& v) {
// The refcount is positive, so the value is refcounted. We need to
// either decref or release.
ifThen(v, CC_NE, sf, [&] (Vout& v) {
// The refcount is greater than 1; decref it.
emitDecRefCount(v, data);
v << ret{live};
});
destroy(v);
});
}
// Either we did a decref, or the value was static.
v << ret{live};
});
}
void cgLdInitRDSAddr(IRLS& env, const IRInstruction* inst) {
auto const dst = dstLoc(env, inst, 0).reg();
auto& v = vmain(env);
ldRDSAddrImpl(v, inst->extra<LdInitRDSAddr>()->handle, dst);
auto const sf = v.makeReg();
emitCmpTVType(v, sf, KindOfUninit, dst[TVOFF(m_type)]);
v << jcc{CC_Z, sf, {label(env, inst->next()), label(env, inst->taken())}};
}
void cgLdInitPropAddr(IRLS& env, const IRInstruction* inst) {
auto const dstLoc = irlower::dstLoc(env, inst, 0);
auto const obj = srcLoc(env, inst, 0).reg();
auto& v = vmain(env);
auto const propPtr = obj[inst->extra<LdInitPropAddr>()->offsetBytes];
if (dstLoc.hasReg(tv_lval::val_idx)) {
v << lea{propPtr, dstLoc.reg(tv_lval::val_idx)};
}
if (wide_tv_val && dstLoc.hasReg(tv_lval::type_idx)) {
static_assert(TVOFF(m_data) == 0, "");
v << lea{propPtr + TVOFF(m_type), dstLoc.reg(tv_lval::type_idx)};
}
auto const sf = v.makeReg();
emitCmpTVType(v, sf, KindOfUninit, propPtr + TVOFF(m_type));
v << jcc{CC_Z, sf, {label(env, inst->next()), label(env, inst->taken())}};
}
void cgCheckInitMem(IRLS& env, const IRInstruction* inst) {
assertx(inst->taken());
auto const src = inst->src(0);
if (!src->type().deref().maybe(TUninit)) return;
auto const ptr = srcLoc(env, inst, 0).reg();
auto& v = vmain(env);
auto const sf = v.makeReg();
emitCmpTVType(v, sf, KindOfUninit, ptr[TVOFF(m_type)]);
v << jcc{CC_Z, sf, {label(env, inst->next()), label(env, inst->taken())}};
}
/*
* Helper for the freeLocalsHelpers which does the actual work of decrementing
* a value's refcount or releasing it.
*
* This helper is reached via call from the various freeLocalHelpers. It
* expects `tv' to be the address of a TypedValue with refcounted type `type'
* (though it may be static, and we will do nothing in that case).
*
* The `live' registers must be preserved across any native calls (and
* generally left untouched).
*/
static TCA emitDecRefHelper(CodeBlock& cb, DataBlock& data, CGMeta& fixups,
PhysReg tv, PhysReg type, RegSet live) {
return vwrap(cb, data, fixups, [&] (Vout& v) {
// Set up frame linkage to avoid an indirect fixup.
v << pushp{rlr(), rfp()};
v << copy{rsp(), rfp()};
// We use the first argument register for the TV data because we might pass
// it to the native release call. It's not live when we enter the helper.
auto const data = rarg(0);
v << load{tv[TVOFF(m_data)], data};
auto const sf = v.makeReg();
v << cmplim{1, data[FAST_REFCOUNT_OFFSET], sf};
ifThen(v, CC_NL, sf, [&] (Vout& v) {
// The refcount is positive, so the value is refcounted. We need to
// either decref or release.
ifThen(v, CC_NE, sf, [&] (Vout& v) {
// The refcount is greater than 1; decref it.
v << declm{data[FAST_REFCOUNT_OFFSET], v.makeReg()};
// Pop FP/LR and return
v << popp{rfp(), rlr()};
v << ret{live};
});
// Note that the stack is aligned since we called to this helper from an
// stack-unaligned stub.
PhysRegSaver prs{v, live};
// The refcount is exactly 1; release the value.
// Avoid 'this' pointer overwriting by reserving it as an argument.
v << callm{lookupDestructor(v, type), arg_regs(1)};
// Between where %rsp is now and the saved RIP of the call into the
// freeLocalsHelpers stub, we have all the live regs we pushed, plus the
// saved RIP of the call from the stub to this helper.
v << syncpoint{makeIndirectFixup(prs.dwordsPushed())};
// fallthru
});
// Either we did a decref, or the value was static.
// Pop FP/LR and return
v << popp{rfp(), rlr()};
v << ret{live};
});
}
/*
* Helper for the freeLocalsHelpers which does the actual work of decrementing
* a value's refcount or releasing it.
*
* This helper is reached via call from the various freeLocalHelpers. It
* expects `tv' to be the address of a TypedValue with refcounted type `type'
* (though it may be static, and we will do nothing in that case).
*
* The `live' registers must be preserved across any native calls (and
* generally left untouched).
*/
static TCA emitDecRefHelper(CodeBlock& cb, DataBlock& data, CGMeta& fixups,
PhysReg tv, PhysReg type, RegSet live) {
return vwrap(cb, data, fixups, [&] (Vout& v) {
// Set up frame linkage to avoid an indirect fixup.
v << stublogue{true};
v << copy{rsp(), rfp()};
// We use the first argument register for the TV data because we might pass
// it to the native release call. It's not live when we enter the helper.
auto const data = rarg(0);
v << load{tv[TVOFF(m_data)], data};
auto destroy = [&](Vout& v) {
// Note that the stack is aligned since we called to this helper from an
// stack-unaligned stub.
PhysRegSaver prs{v, live};
// The refcount is exactly 1; release the value.
// Avoid 'this' pointer overwriting by reserving it as an argument.
// There's no need for a fixup, because we setup a frame on the c++
// stack.
v << callm{lookupDestructor(v, type), arg_regs(1)};
// fallthru
};
auto const sf = emitCmpRefCount(v, OneReference, data);
if (one_bit_refcount) {
ifThen(v, CC_E, sf, destroy);
} else {
ifThen(v, CC_NL, sf, [&] (Vout& v) {
// The refcount is positive, so the value is refcounted. We need to
// either decref or release.
ifThen(v, CC_NE, sf, [&] (Vout& v) {
// The refcount is greater than 1; decref it.
emitDecRefCount(v, data);
v << stubret{live, true};
});
destroy(v);
});
}
// Either we did a decref, or the value was static.
v << stubret{live, true};
});
}
TCA emitFreeLocalsHelpers(CodeBlock& cb, DataBlock& data, UniqueStubs& us) {
// The address of the first local is passed in the second argument register.
// We use the third and fourth as scratch registers.
auto const local = rarg(1);
auto const last = rarg(2);
auto const type = rarg(3);
CGMeta fixups;
TCA freeLocalsHelpers[kNumFreeLocalsHelpers];
TCA freeManyLocalsHelper;
// This stub is very hot; keep it cache-aligned.
align(cb, &fixups, Alignment::CacheLine, AlignContext::Dead);
auto const release =
emitDecRefHelper(cb, data, fixups, local, type, local | last);
auto const decref_local = [&] (Vout& v) {
auto const sf = v.makeReg();
// We can't use emitLoadTVType() here because it does a byte load, and we
// need to sign-extend since we use `type' as a 32-bit array index to the
// destructor table.
v << loadzbl{local[TVOFF(m_type)], type};
emitCmpTVType(v, sf, KindOfRefCountThreshold, type);
ifThen(v, CC_G, sf, [&] (Vout& v) {
v << call{release, arg_regs(3)};
});
};
auto const next_local = [&] (Vout& v) {
v << addqi{static_cast<int>(sizeof(TypedValue)),
local, local, v.makeReg()};
};
alignJmpTarget(cb);
freeManyLocalsHelper = vwrap(cb, data, [&] (Vout& v) {
// We always unroll the final `kNumFreeLocalsHelpers' decrefs, so only loop
// until we hit that point.
v << lea{rvmfp()[localOffset(kNumFreeLocalsHelpers - 1)], last};
// Set up frame linkage to avoid an indirect fixup.
v << copy{rsp(), rfp()};
doWhile(v, CC_NZ, {},
[&] (const VregList& in, const VregList& out) {
auto const sf = v.makeReg();
decref_local(v);
next_local(v);
v << cmpq{local, last, sf};
return sf;
}
);
});
for (auto i = kNumFreeLocalsHelpers - 1; i >= 0; --i) {
freeLocalsHelpers[i] = vwrap(cb, data, [&] (Vout& v) {
decref_local(v);
if (i != 0) next_local(v);
});
}
// All the stub entrypoints share the same ret.
vwrap(cb, data, fixups, [] (Vout& v) {
v << popp{rfp(), rlr()};
v << ret{};
});
// Create a table of branches
us.freeManyLocalsHelper = vwrap(cb, data, [&] (Vout& v) {
v << pushp{rlr(), rfp()};
// rvmfp() is needed by the freeManyLocalsHelper stub above, so frame
// linkage setup is deferred until after its use in freeManyLocalsHelper.
v << jmpi{freeManyLocalsHelper};
});
for (auto i = kNumFreeLocalsHelpers - 1; i >= 0; --i) {
us.freeLocalsHelpers[i] = vwrap(cb, data, [&] (Vout& v) {
// We set up frame linkage to avoid an indirect fixup.
v << pushp{rlr(), rfp()};
v << copy{rsp(), rfp()};
v << jmpi{freeLocalsHelpers[i]};
});
}
// FIXME: This stub is hot, so make sure to keep it small.
#if 0
always_assert(Stats::enabled() ||
(cb.frontier() - release <= 4 * x64::cache_line_size()));
#endif
fixups.process(nullptr);
return release;
}
void cgCallBuiltin(IRLS& env, const IRInstruction* inst) {
auto const extra = inst->extra<CallBuiltin>();
auto const callee = extra->callee;
auto const returnType = inst->typeParam();
auto const funcReturnType = callee->returnType();
auto const returnByValue = callee->isReturnByValue();
auto const dstData = dstLoc(env, inst, 0).reg(0);
auto const dstType = dstLoc(env, inst, 0).reg(1);
auto& v = vmain(env);
// Whether `t' is passed in/out of C++ as String&/Array&/Object&.
auto const isReqPtrRef = [] (MaybeDataType t) {
return isStringType(t) || isArrayLikeType(t) ||
t == KindOfObject || t == KindOfResource;
};
if (FixupMap::eagerRecord(callee)) {
auto const sp = srcLoc(env, inst, 1).reg();
auto const spOffset = cellsToBytes(extra->spOffset.offset);
auto const& marker = inst->marker();
auto const pc = marker.fixupSk().unit()->entry() + marker.fixupBcOff();
auto const synced_sp = v.makeReg();
v << lea{sp[spOffset], synced_sp};
emitEagerSyncPoint(v, pc, rvmtl(), srcLoc(env, inst, 0).reg(), synced_sp);
}
int returnOffset = rds::kVmMInstrStateOff +
offsetof(MInstrState, tvBuiltinReturn);
auto args = argGroup(env, inst);
if (!returnByValue) {
if (isBuiltinByRef(funcReturnType)) {
if (isReqPtrRef(funcReturnType)) {
returnOffset += TVOFF(m_data);
}
// Pass the address of tvBuiltinReturn to the native function as the
// location where it can construct the return Array, String, Object, or
// Variant.
args.addr(rvmtl(), returnOffset);
args.indirect();
}
}
// The srcs past the first two (sp and fp) are the arguments to the callee.
auto srcNum = uint32_t{2};
// Add the this_ or self_ argument for HNI builtins.
if (callee->isMethod()) {
if (callee->isStatic()) {
args.ssa(srcNum);
++srcNum;
} else {
// Note that we don't support objects with vtables here (if they may need
// a $this pointer adjustment). This should be filtered out during irgen
// or before.
args.ssa(srcNum);
++srcNum;
}
}
// Add the func_num_args() value if needed.
if (callee->attrs() & AttrNumArgs) {
// If `numNonDefault' is negative, this is passed as an src.
if (extra->numNonDefault >= 0) {
args.imm((int64_t)extra->numNonDefault);
} else {
args.ssa(srcNum);
++srcNum;
}
}
// Add the positional arguments.
for (uint32_t i = 0; i < callee->numParams(); ++i, ++srcNum) {
auto const& pi = callee->params()[i];
// Non-pointer and NativeArg args are passed by value. String, Array,
// Object, and Variant are passed by const&, i.e. a pointer to stack memory
// holding the value, so we expect PtrToT types for these. Pointers to
// req::ptr types (String, Array, Object) need adjusting to point to
// &ptr->m_data.
if (TVOFF(m_data) && !pi.nativeArg && isReqPtrRef(pi.builtinType)) {
assertx(inst->src(srcNum)->type() <= TPtrToGen);
args.addr(srcLoc(env, inst, srcNum).reg(), TVOFF(m_data));
} else if (pi.nativeArg && !pi.builtinType && !callee->byRef(i)) {
// This condition indicates a MixedTV (i.e., TypedValue-by-value) arg.
args.typedValue(srcNum);
} else {
args.ssa(srcNum, pi.builtinType == KindOfDouble);
}
}
auto dest = [&] () -> CallDest {
if (isBuiltinByRef(funcReturnType)) {
if (!returnByValue) return kVoidDest; // indirect return
return funcReturnType
? callDest(dstData) // String, Array, or Object
: callDest(dstData, dstType); // Variant
}
return funcReturnType == KindOfDouble
? callDestDbl(env, inst)
: callDest(env, inst);
}();
cgCallHelper(v, env, CallSpec::direct(callee->nativeFuncPtr()),
dest, SyncOptions::Sync, args);
// For primitive return types (int, bool, double) and returnByValue, the
// return value is already in dstData/dstType.
if (returnType.isSimpleType() || returnByValue) return;
// For return by reference (String, Object, Array, Variant), the builtin
// writes the return value into MInstrState::tvBuiltinReturn, from where it
// has to be tested and copied.
if (returnType.isReferenceType()) {
// The return type is String, Array, or Object; fold nullptr to KindOfNull.
assertx(isBuiltinByRef(funcReturnType) && isReqPtrRef(funcReturnType));
v << load{rvmtl()[returnOffset], dstData};
if (dstType.isValid()) {
auto const sf = v.makeReg();
auto const rtype = v.cns(returnType.toDataType());
auto const nulltype = v.cns(KindOfNull);
v << testq{dstData, dstData, sf};
v << cmovb{CC_Z, sf, rtype, nulltype, dstType};
}
return;
}
if (returnType <= TCell || returnType <= TBoxedCell) {
// The return type is Variant; fold KindOfUninit to KindOfNull.
assertx(isBuiltinByRef(funcReturnType) && !isReqPtrRef(funcReturnType));
static_assert(KindOfUninit == 0, "KindOfUninit must be 0 for test");
v << load{rvmtl()[returnOffset + TVOFF(m_data)], dstData};
if (dstType.isValid()) {
auto const rtype = v.makeReg();
v << loadb{rvmtl()[returnOffset + TVOFF(m_type)], rtype};
auto const sf = v.makeReg();
auto const nulltype = v.cns(KindOfNull);
v << testb{rtype, rtype, sf};
v << cmovb{CC_Z, sf, rtype, nulltype, dstType};
}
return;
}
not_reached();
}
TCA emitFreeLocalsHelpers(CodeBlock& cb, UniqueStubs& us) {
// The address of the first local is passed in the second argument register.
// We use the third and fourth as scratch registers.
auto const local = rarg(1);
auto const last = rarg(2);
auto const type = rarg(3);
CGMeta fixups;
// This stub is very hot; keep it cache-aligned.
align(cb, &fixups, Alignment::CacheLine, AlignContext::Dead);
auto const release = emitDecRefHelper(cb, fixups, local, type, local | last);
auto const decref_local = [&] (Vout& v) {
auto const sf = v.makeReg();
// We can't do a byte load here---we have to sign-extend since we use
// `type' as a 32-bit array index to the destructor table.
v << loadzbl{local[TVOFF(m_type)], type};
emitCmpTVType(v, sf, KindOfRefCountThreshold, type);
ifThen(v, CC_G, sf, [&] (Vout& v) {
v << call{release, arg_regs(3)};
});
};
auto const next_local = [&] (Vout& v) {
v << addqi{static_cast<int>(sizeof(TypedValue)),
local, local, v.makeReg()};
};
alignJmpTarget(cb);
us.freeManyLocalsHelper = vwrap(cb, fixups, [&] (Vout& v) {
// We always unroll the final `kNumFreeLocalsHelpers' decrefs, so only loop
// until we hit that point.
v << lea{rvmfp()[localOffset(kNumFreeLocalsHelpers - 1)], last};
doWhile(v, CC_NZ, {},
[&] (const VregList& in, const VregList& out) {
auto const sf = v.makeReg();
decref_local(v);
next_local(v);
v << cmpq{local, last, sf};
return sf;
}
);
});
for (auto i = kNumFreeLocalsHelpers - 1; i >= 0; --i) {
us.freeLocalsHelpers[i] = vwrap(cb, [&] (Vout& v) {
decref_local(v);
if (i != 0) next_local(v);
});
}
// All the stub entrypoints share the same ret.
vwrap(cb, fixups, [] (Vout& v) { v << ret{}; });
// This stub is hot, so make sure to keep it small.
// Alas, we have more work to do in this under Windows,
// so we can't be this small :(
#ifndef _WIN32
always_assert(Stats::enabled() ||
(cb.frontier() - release <= 4 * x64::cache_line_size()));
#endif
fixups.process(nullptr);
return release;
}
TCA emitFreeLocalsHelpers(CodeBlock& cb, DataBlock& data, UniqueStubs& us) {
// The address of the first local is passed in the second argument register.
// We use the third and fourth as scratch registers.
auto const local = rarg(1);
auto const last = rarg(2);
auto const type = rarg(3);
CGMeta fixups;
// This stub is very hot; keep it cache-aligned.
align(cb, &fixups, Alignment::CacheLine, AlignContext::Dead);
auto const release =
emitDecRefHelper(cb, data, fixups, local, type, local | last);
auto const decref_local = [&] (Vout& v) {
auto const sf = v.makeReg();
// We can't do a byte load here---we have to sign-extend since we use
// `type' as a 32-bit array index to the destructor table.
v << loadzbl{local[TVOFF(m_type)], type};
emitCmpTVType(v, sf, KindOfRefCountThreshold, type);
ifThen(v, CC_G, sf, [&] (Vout& v) {
auto const dword_size = sizeof(int64_t);
// saving return value on the stack, but keeping it 16-byte aligned
v << mflr{rfuncln()};
v << lea {rsp()[-2 * dword_size], rsp()};
v << store{rfuncln(), rsp()[0]};
v << call{release, arg_regs(3)};
// restore the return value from the stack
v << load{rsp()[0], rfuncln()};
v << lea {rsp()[2 * dword_size], rsp()};
v << mtlr{rfuncln()};
});
};
auto const next_local = [&] (Vout& v) {
v << addqi{static_cast<int>(sizeof(TypedValue)),
local, local, v.makeReg()};
};
alignJmpTarget(cb);
us.freeManyLocalsHelper = vwrap(cb, data, fixups, [&] (Vout& v) {
// We always unroll the final `kNumFreeLocalsHelpers' decrefs, so only loop
// until we hit that point.
v << lea{rvmfp()[localOffset(kNumFreeLocalsHelpers - 1)], last};
doWhile(v, CC_NZ, {},
[&] (const VregList& in, const VregList& out) {
auto const sf = v.makeReg();
decref_local(v);
next_local(v);
v << cmpq{local, last, sf};
return sf;
}
);
});
for (auto i = kNumFreeLocalsHelpers - 1; i >= 0; --i) {
us.freeLocalsHelpers[i] = vwrap(cb, data, [&] (Vout& v) {
decref_local(v);
if (i != 0) next_local(v);
});
}
// All the stub entrypoints share the same ret.
vwrap(cb, data, fixups, [] (Vout& v) { v << ret{}; });
// This stub is hot, so make sure to keep it small.
#if 0
// TODO(gut): Currently this assert fails.
// Take a closer look when looking at performance
always_assert(Stats::enabled() ||
(cb.frontier() - release <= 4 * cache_line_size()));
#endif
fixups.process(nullptr);
return release;
}
void cgCheckTypeMem(IRLS& env, const IRInstruction* inst) {
auto const src = srcLoc(env, inst, 0).reg();
emitTypeCheck(vmain(env), env, inst->typeParam(),
src[TVOFF(m_type)], src[TVOFF(m_data)], inst->taken());
}
