/*
* Helper for the freeLocalsHelpers which does the actual work of decrementing
* a value's refcount or releasing it.
*
* This helper is reached via call from the various freeLocalHelpers. It
* expects `tv' to be the address of a TypedValue with refcounted type `type'
* (though it may be static, and we will do nothing in that case).
*
* The `live' registers must be preserved across any native calls (and
* generally left untouched).
*/
static TCA emitDecRefHelper(CodeBlock& cb, DataBlock& data, CGMeta& fixups,
PhysReg tv, PhysReg type, RegSet live) {
return vwrap(cb, data, fixups, [&] (Vout& v) {
// We use the first argument register for the TV data because we might pass
// it to the native release call. It's not live when we enter the helper.
auto const data = rarg(0);
v << load{tv[TVOFF(m_data)], data};
auto destroy = [&](Vout& v) {
PhysRegSaver prs{v, live};
auto const dword_size = sizeof(int64_t);
// saving return value on the stack, but keeping it 16-byte aligned
v << mflr{rfuncln()};
v << lea {rsp()[-2 * dword_size], rsp()};
v << store{rfuncln(), rsp()[0]};
// The refcount is exactly 1; release the value.
// Avoid 'this' pointer overwriting by reserving it as an argument.
v << callm{lookupDestructor(v, type), arg_regs(1)};
// Between where r1 is now and the saved RIP of the call into the
// freeLocalsHelpers stub, we have all the live regs we pushed, plus the
// stack size reserved for the LR saved right above and the LR offset in
// the frame.
v << syncpoint{makeIndirectFixup(prs.dwordsPushed())};
// fallthru
// restore the return value from the stack
v << load{rsp()[0], rfuncln()};
v << lea {rsp()[2 * dword_size], rsp()};
v << mtlr{rfuncln()};
};
auto const sf = emitCmpRefCount(v, OneReference, data);
if (one_bit_refcount) {
ifThen(v, CC_E, sf, destroy);
} else {
ifThen(v, CC_NL, sf, [&] (Vout& v) {
// The refcount is positive, so the value is refcounted. We need to
// either decref or release.
ifThen(v, CC_NE, sf, [&] (Vout& v) {
// The refcount is greater than 1; decref it.
emitDecRefCount(v, data);
v << ret{live};
});
destroy(v);
});
}
// Either we did a decref, or the value was static.
v << ret{live};
});
}
TCA emitEndCatchHelper(CodeBlock& cb, UniqueStubs& us) {
auto const udrspo = rvmtl()[unwinderDebuggerReturnSPOff()];
auto const debuggerReturn = vwrap(cb, [&] (Vout& v) {
v << load{udrspo, rvmsp()};
v << storeqi{0, udrspo};
});
svcreq::emit_persistent(cb, folly::none, REQ_POST_DEBUGGER_RET);
auto const resumeCPPUnwind = vwrap(cb, [] (Vout& v) {
static_assert(sizeof(tl_regState) == 1,
"The following store must match the size of tl_regState.");
auto const regstate = emitTLSAddr(v, tls_datum(tl_regState));
v << storebi{static_cast<int32_t>(VMRegState::CLEAN), regstate};
v << load{rvmtl()[unwinderExnOff()], rarg(0)};
v << call{TCA(_Unwind_Resume), arg_regs(1)};
});
us.endCatchHelperPast = cb.frontier();
vwrap(cb, [] (Vout& v) { v << ud2{}; });
alignJmpTarget(cb);
return vwrap(cb, [&] (Vout& v) {
auto const done1 = v.makeBlock();
auto const sf1 = v.makeReg();
v << cmpqim{0, udrspo, sf1};
v << jcci{CC_NE, sf1, done1, debuggerReturn};
v = done1;
// Normal end catch situation: call back to tc_unwind_resume, which returns
// the catch trace (or null) in %rax, and the new vmfp in %rdx.
v << copy{rvmfp(), rarg(0)};
v << call{TCA(tc_unwind_resume)};
v << copy{reg::rdx, rvmfp()};
auto const done2 = v.makeBlock();
auto const sf2 = v.makeReg();
v << testq{reg::rax, reg::rax, sf2};
v << jcci{CC_Z, sf2, done2, resumeCPPUnwind};
v = done2;
// We need to do a syncForLLVMCatch(), but vmfp is already in rdx.
v << jmpr{reg::rax};
});
}
/*
* Helper for the freeLocalsHelpers which does the actual work of decrementing
* a value's refcount or releasing it.
*
* This helper is reached via call from the various freeLocalHelpers. It
* expects `tv' to be the address of a TypedValue with refcounted type `type'
* (though it may be static, and we will do nothing in that case).
*
* The `live' registers must be preserved across any native calls (and
* generally left untouched).
*/
static TCA emitDecRefHelper(CodeBlock& cb, DataBlock& data, CGMeta& fixups,
PhysReg tv, PhysReg type, RegSet live) {
return vwrap(cb, data, fixups, [&] (Vout& v) {
// Set up frame linkage to avoid an indirect fixup.
v << pushp{rlr(), rfp()};
v << copy{rsp(), rfp()};
// We use the first argument register for the TV data because we might pass
// it to the native release call. It's not live when we enter the helper.
auto const data = rarg(0);
v << load{tv[TVOFF(m_data)], data};
auto const sf = v.makeReg();
v << cmplim{1, data[FAST_REFCOUNT_OFFSET], sf};
ifThen(v, CC_NL, sf, [&] (Vout& v) {
// The refcount is positive, so the value is refcounted. We need to
// either decref or release.
ifThen(v, CC_NE, sf, [&] (Vout& v) {
// The refcount is greater than 1; decref it.
v << declm{data[FAST_REFCOUNT_OFFSET], v.makeReg()};
// Pop FP/LR and return
v << popp{rfp(), rlr()};
v << ret{live};
});
// Note that the stack is aligned since we called to this helper from an
// stack-unaligned stub.
PhysRegSaver prs{v, live};
// The refcount is exactly 1; release the value.
// Avoid 'this' pointer overwriting by reserving it as an argument.
v << callm{lookupDestructor(v, type), arg_regs(1)};
// Between where %rsp is now and the saved RIP of the call into the
// freeLocalsHelpers stub, we have all the live regs we pushed, plus the
// saved RIP of the call from the stub to this helper.
v << syncpoint{makeIndirectFixup(prs.dwordsPushed())};
// fallthru
});
// Either we did a decref, or the value was static.
// Pop FP/LR and return
v << popp{rfp(), rlr()};
v << ret{live};
});
}
/*
* Helper for the freeLocalsHelpers which does the actual work of decrementing
* a value's refcount or releasing it.
*
* This helper is reached via call from the various freeLocalHelpers. It
* expects `tv' to be the address of a TypedValue with refcounted type `type'
* (though it may be static, and we will do nothing in that case).
*
* The `live' registers must be preserved across any native calls (and
* generally left untouched).
*/
static TCA emitDecRefHelper(CodeBlock& cb, DataBlock& data, CGMeta& fixups,
PhysReg tv, PhysReg type, RegSet live) {
return vwrap(cb, data, fixups, [&] (Vout& v) {
// Set up frame linkage to avoid an indirect fixup.
v << stublogue{true};
v << copy{rsp(), rfp()};
// We use the first argument register for the TV data because we might pass
// it to the native release call. It's not live when we enter the helper.
auto const data = rarg(0);
v << load{tv[TVOFF(m_data)], data};
auto destroy = [&](Vout& v) {
// Note that the stack is aligned since we called to this helper from an
// stack-unaligned stub.
PhysRegSaver prs{v, live};
// The refcount is exactly 1; release the value.
// Avoid 'this' pointer overwriting by reserving it as an argument.
// There's no need for a fixup, because we setup a frame on the c++
// stack.
v << callm{lookupDestructor(v, type), arg_regs(1)};
// fallthru
};
auto const sf = emitCmpRefCount(v, OneReference, data);
if (one_bit_refcount) {
ifThen(v, CC_E, sf, destroy);
} else {
ifThen(v, CC_NL, sf, [&] (Vout& v) {
// The refcount is positive, so the value is refcounted. We need to
// either decref or release.
ifThen(v, CC_NE, sf, [&] (Vout& v) {
// The refcount is greater than 1; decref it.
emitDecRefCount(v, data);
v << stubret{live, true};
});
destroy(v);
});
}
// Either we did a decref, or the value was static.
v << stubret{live, true};
});
}
TCA emitFreeLocalsHelpers(CodeBlock& cb, UniqueStubs& us) {
// The address of the first local is passed in the second argument register.
// We use the third and fourth as scratch registers.
auto const local = rarg(1);
auto const last = rarg(2);
auto const type = rarg(3);
CGMeta fixups;
// This stub is very hot; keep it cache-aligned.
align(cb, &fixups, Alignment::CacheLine, AlignContext::Dead);
auto const release = emitDecRefHelper(cb, fixups, local, type, local | last);
auto const decref_local = [&] (Vout& v) {
auto const sf = v.makeReg();
// We can't do a byte load here---we have to sign-extend since we use
// `type' as a 32-bit array index to the destructor table.
v << loadzbl{local[TVOFF(m_type)], type};
emitCmpTVType(v, sf, KindOfRefCountThreshold, type);
ifThen(v, CC_G, sf, [&] (Vout& v) {
v << call{release, arg_regs(3)};
});
};
auto const next_local = [&] (Vout& v) {
v << addqi{static_cast<int>(sizeof(TypedValue)),
local, local, v.makeReg()};
};
alignJmpTarget(cb);
us.freeManyLocalsHelper = vwrap(cb, fixups, [&] (Vout& v) {
// We always unroll the final `kNumFreeLocalsHelpers' decrefs, so only loop
// until we hit that point.
v << lea{rvmfp()[localOffset(kNumFreeLocalsHelpers - 1)], last};
doWhile(v, CC_NZ, {},
[&] (const VregList& in, const VregList& out) {
auto const sf = v.makeReg();
decref_local(v);
next_local(v);
v << cmpq{local, last, sf};
return sf;
}
);
});
for (auto i = kNumFreeLocalsHelpers - 1; i >= 0; --i) {
us.freeLocalsHelpers[i] = vwrap(cb, [&] (Vout& v) {
decref_local(v);
if (i != 0) next_local(v);
});
}
// All the stub entrypoints share the same ret.
vwrap(cb, fixups, [] (Vout& v) { v << ret{}; });
// This stub is hot, so make sure to keep it small.
// Alas, we have more work to do in this under Windows,
// so we can't be this small :(
#ifndef _WIN32
always_assert(Stats::enabled() ||
(cb.frontier() - release <= 4 * x64::cache_line_size()));
#endif
fixups.process(nullptr);
return release;
}
TCA emitFreeLocalsHelpers(CodeBlock& cb, DataBlock& data, UniqueStubs& us) {
// The address of the first local is passed in the second argument register.
// We use the third and fourth as scratch registers.
auto const local = rarg(1);
auto const last = rarg(2);
auto const type = rarg(3);
CGMeta fixups;
// This stub is very hot; keep it cache-aligned.
align(cb, &fixups, Alignment::CacheLine, AlignContext::Dead);
auto const release =
emitDecRefHelper(cb, data, fixups, local, type, local | last);
auto const decref_local = [&] (Vout& v) {
auto const sf = v.makeReg();
// We can't do a byte load here---we have to sign-extend since we use
// `type' as a 32-bit array index to the destructor table.
v << loadzbl{local[TVOFF(m_type)], type};
emitCmpTVType(v, sf, KindOfRefCountThreshold, type);
ifThen(v, CC_G, sf, [&] (Vout& v) {
auto const dword_size = sizeof(int64_t);
// saving return value on the stack, but keeping it 16-byte aligned
v << mflr{rfuncln()};
v << lea {rsp()[-2 * dword_size], rsp()};
v << store{rfuncln(), rsp()[0]};
v << call{release, arg_regs(3)};
// restore the return value from the stack
v << load{rsp()[0], rfuncln()};
v << lea {rsp()[2 * dword_size], rsp()};
v << mtlr{rfuncln()};
});
};
auto const next_local = [&] (Vout& v) {
v << addqi{static_cast<int>(sizeof(TypedValue)),
local, local, v.makeReg()};
};
alignJmpTarget(cb);
us.freeManyLocalsHelper = vwrap(cb, data, fixups, [&] (Vout& v) {
// We always unroll the final `kNumFreeLocalsHelpers' decrefs, so only loop
// until we hit that point.
v << lea{rvmfp()[localOffset(kNumFreeLocalsHelpers - 1)], last};
doWhile(v, CC_NZ, {},
[&] (const VregList& in, const VregList& out) {
auto const sf = v.makeReg();
decref_local(v);
next_local(v);
v << cmpq{local, last, sf};
return sf;
}
);
});
for (auto i = kNumFreeLocalsHelpers - 1; i >= 0; --i) {
us.freeLocalsHelpers[i] = vwrap(cb, data, [&] (Vout& v) {
decref_local(v);
if (i != 0) next_local(v);
});
}
// All the stub entrypoints share the same ret.
vwrap(cb, data, fixups, [] (Vout& v) { v << ret{}; });
// This stub is hot, so make sure to keep it small.
#if 0
// TODO(gut): Currently this assert fails.
// Take a closer look when looking at performance
always_assert(Stats::enabled() ||
(cb.frontier() - release <= 4 * cache_line_size()));
#endif
fixups.process(nullptr);
return release;
}
TCA emitFreeLocalsHelpers(CodeBlock& cb, DataBlock& data, UniqueStubs& us) {
// The address of the first local is passed in the second argument register.
// We use the third and fourth as scratch registers.
auto const local = rarg(1);
auto const last = rarg(2);
auto const type = rarg(3);
CGMeta fixups;
TCA freeLocalsHelpers[kNumFreeLocalsHelpers];
TCA freeManyLocalsHelper;
// This stub is very hot; keep it cache-aligned.
align(cb, &fixups, Alignment::CacheLine, AlignContext::Dead);
auto const release =
emitDecRefHelper(cb, data, fixups, local, type, local | last);
auto const decref_local = [&] (Vout& v) {
auto const sf = v.makeReg();
// We can't use emitLoadTVType() here because it does a byte load, and we
// need to sign-extend since we use `type' as a 32-bit array index to the
// destructor table.
v << loadzbl{local[TVOFF(m_type)], type};
emitCmpTVType(v, sf, KindOfRefCountThreshold, type);
ifThen(v, CC_G, sf, [&] (Vout& v) {
v << call{release, arg_regs(3)};
});
};
auto const next_local = [&] (Vout& v) {
v << addqi{static_cast<int>(sizeof(TypedValue)),
local, local, v.makeReg()};
};
alignJmpTarget(cb);
freeManyLocalsHelper = vwrap(cb, data, [&] (Vout& v) {
// We always unroll the final `kNumFreeLocalsHelpers' decrefs, so only loop
// until we hit that point.
v << lea{rvmfp()[localOffset(kNumFreeLocalsHelpers - 1)], last};
// Set up frame linkage to avoid an indirect fixup.
v << copy{rsp(), rfp()};
doWhile(v, CC_NZ, {},
[&] (const VregList& in, const VregList& out) {
auto const sf = v.makeReg();
decref_local(v);
next_local(v);
v << cmpq{local, last, sf};
return sf;
}
);
});
for (auto i = kNumFreeLocalsHelpers - 1; i >= 0; --i) {
freeLocalsHelpers[i] = vwrap(cb, data, [&] (Vout& v) {
decref_local(v);
if (i != 0) next_local(v);
});
}
// All the stub entrypoints share the same ret.
vwrap(cb, data, fixups, [] (Vout& v) {
v << popp{rfp(), rlr()};
v << ret{};
});
// Create a table of branches
us.freeManyLocalsHelper = vwrap(cb, data, [&] (Vout& v) {
v << pushp{rlr(), rfp()};
// rvmfp() is needed by the freeManyLocalsHelper stub above, so frame
// linkage setup is deferred until after its use in freeManyLocalsHelper.
v << jmpi{freeManyLocalsHelper};
});
for (auto i = kNumFreeLocalsHelpers - 1; i >= 0; --i) {
us.freeLocalsHelpers[i] = vwrap(cb, data, [&] (Vout& v) {
// We set up frame linkage to avoid an indirect fixup.
v << pushp{rlr(), rfp()};
v << copy{rsp(), rfp()};
v << jmpi{freeLocalsHelpers[i]};
});
}
// FIXME: This stub is hot, so make sure to keep it small.
#if 0
always_assert(Stats::enabled() ||
(cb.frontier() - release <= 4 * x64::cache_line_size()));
#endif
fixups.process(nullptr);
return release;
}
