/* * Helper for the freeLocalsHelpers which does the actual work of decrementing * a value's refcount or releasing it. * * This helper is reached via call from the various freeLocalHelpers. It * expects `tv' to be the address of a TypedValue with refcounted type `type' * (though it may be static, and we will do nothing in that case). * * The `live' registers must be preserved across any native calls (and * generally left untouched). */ static TCA emitDecRefHelper(CodeBlock& cb, DataBlock& data, CGMeta& fixups, PhysReg tv, PhysReg type, RegSet live) { return vwrap(cb, data, fixups, [&] (Vout& v) { // We use the first argument register for the TV data because we might pass // it to the native release call. It's not live when we enter the helper. auto const data = rarg(0); v << load{tv[TVOFF(m_data)], data}; auto destroy = [&](Vout& v) { PhysRegSaver prs{v, live}; auto const dword_size = sizeof(int64_t); // saving return value on the stack, but keeping it 16-byte aligned v << mflr{rfuncln()}; v << lea {rsp()[-2 * dword_size], rsp()}; v << store{rfuncln(), rsp()[0]}; // The refcount is exactly 1; release the value. // Avoid 'this' pointer overwriting by reserving it as an argument. v << callm{lookupDestructor(v, type), arg_regs(1)}; // Between where r1 is now and the saved RIP of the call into the // freeLocalsHelpers stub, we have all the live regs we pushed, plus the // stack size reserved for the LR saved right above and the LR offset in // the frame. v << syncpoint{makeIndirectFixup(prs.dwordsPushed())}; // fallthru // restore the return value from the stack v << load{rsp()[0], rfuncln()}; v << lea {rsp()[2 * dword_size], rsp()}; v << mtlr{rfuncln()}; }; auto const sf = emitCmpRefCount(v, OneReference, data); if (one_bit_refcount) { ifThen(v, CC_E, sf, destroy); } else { ifThen(v, CC_NL, sf, [&] (Vout& v) { // The refcount is positive, so the value is refcounted. We need to // either decref or release. ifThen(v, CC_NE, sf, [&] (Vout& v) { // The refcount is greater than 1; decref it. emitDecRefCount(v, data); v << ret{live}; }); destroy(v); }); } // Either we did a decref, or the value was static. v << ret{live}; }); }
TCA emitFreeLocalsHelpers(CodeBlock& cb, DataBlock& data, UniqueStubs& us) { // The address of the first local is passed in the second argument register. // We use the third and fourth as scratch registers. auto const local = rarg(1); auto const last = rarg(2); auto const type = rarg(3); CGMeta fixups; // This stub is very hot; keep it cache-aligned. align(cb, &fixups, Alignment::CacheLine, AlignContext::Dead); auto const release = emitDecRefHelper(cb, data, fixups, local, type, local | last); auto const decref_local = [&] (Vout& v) { auto const sf = v.makeReg(); // We can't do a byte load here---we have to sign-extend since we use // `type' as a 32-bit array index to the destructor table. v << loadzbl{local[TVOFF(m_type)], type}; emitCmpTVType(v, sf, KindOfRefCountThreshold, type); ifThen(v, CC_G, sf, [&] (Vout& v) { auto const dword_size = sizeof(int64_t); // saving return value on the stack, but keeping it 16-byte aligned v << mflr{rfuncln()}; v << lea {rsp()[-2 * dword_size], rsp()}; v << store{rfuncln(), rsp()[0]}; v << call{release, arg_regs(3)}; // restore the return value from the stack v << load{rsp()[0], rfuncln()}; v << lea {rsp()[2 * dword_size], rsp()}; v << mtlr{rfuncln()}; }); }; auto const next_local = [&] (Vout& v) { v << addqi{static_cast<int>(sizeof(TypedValue)), local, local, v.makeReg()}; }; alignJmpTarget(cb); us.freeManyLocalsHelper = vwrap(cb, data, fixups, [&] (Vout& v) { // We always unroll the final `kNumFreeLocalsHelpers' decrefs, so only loop // until we hit that point. v << lea{rvmfp()[localOffset(kNumFreeLocalsHelpers - 1)], last}; doWhile(v, CC_NZ, {}, [&] (const VregList& in, const VregList& out) { auto const sf = v.makeReg(); decref_local(v); next_local(v); v << cmpq{local, last, sf}; return sf; } ); }); for (auto i = kNumFreeLocalsHelpers - 1; i >= 0; --i) { us.freeLocalsHelpers[i] = vwrap(cb, data, [&] (Vout& v) { decref_local(v); if (i != 0) next_local(v); }); } // All the stub entrypoints share the same ret. vwrap(cb, data, fixups, [] (Vout& v) { v << ret{}; }); // This stub is hot, so make sure to keep it small. #if 0 // TODO(gut): Currently this assert fails. // Take a closer look when looking at performance always_assert(Stats::enabled() || (cb.frontier() - release <= 4 * cache_line_size())); #endif fixups.process(nullptr); return release; }