void emitStaticLocInit(HTS& env, int32_t locId, const StringData* name) { if (curFunc(env)->isPseudoMain()) PUNT(StaticLocInit); auto const ldPMExit = makePseudoMainExit(env); auto const value = popC(env); // Closures and generators from closures don't satisfy the "one static per // source location" rule that the inline fastpath requires auto const box = [&]{ if (curFunc(env)->isClosureBody()) { return gen(env, ClosureStaticLocInit, cns(env, name), fp(env), value); } auto const cachedBox = gen(env, LdStaticLocCached, StaticLocName { curFunc(env), name }); ifThen( env, [&] (Block* taken) { gen(env, CheckStaticLocInit, taken, cachedBox); }, [&] { hint(env, Block::Hint::Unlikely); gen(env, StaticLocInitCached, cachedBox, value); } ); return cachedBox; }(); gen(env, IncRef, box); auto const oldValue = ldLoc(env, locId, ldPMExit, DataTypeSpecific); stLocRaw(env, locId, fp(env), box); gen(env, DecRef, oldValue); // We don't need to decref value---it's a bytecode invariant that // our Cell was not ref-counted. }
void emitInitThisLoc(HTS& env, int32_t id) { if (!curClass(env)) { // Do nothing if this is null return; } auto const ldrefExit = makeExit(env); auto const oldLoc = ldLoc(env, id, ldrefExit, DataTypeCountness); auto const ctx = gen(env, LdCtx, fp(env)); gen(env, CheckCtxThis, makeExitSlow(env), ctx); auto const this_ = gen(env, CastCtxThis, ctx); gen(env, IncRef, this_); stLocRaw(env, id, fp(env), this_); gen(env, DecRef, oldLoc); }
void emitVGetL(HTS& env, int32_t id) { auto value = ldLoc(env, id, makeExit(env), DataTypeCountnessInit); auto const t = value->type(); always_assert(t.isBoxed() || t.notBoxed()); if (t.notBoxed()) { if (value->isA(Type::Uninit)) { value = cns(env, Type::InitNull); } value = gen(env, Box, value); stLocRaw(env, id, fp(env), value); } pushIncRef(env, value); }
void emitBindL(HTS& env, int32_t id) { if (curFunc(env)->isPseudoMain()) { interpOne(env, Type::BoxedInitCell, 1); return; } auto const ldPMExit = makePseudoMainExit(env); auto const newValue = popV(env); // Note that the IncRef must happen first, for correctness in a // pseudo-main: the destructor could decref the value again after // we've stored it into the local. pushIncRef(env, newValue); auto const oldValue = ldLoc(env, id, ldPMExit, DataTypeSpecific); stLocRaw(env, id, fp(env), newValue); gen(env, DecRef, oldValue); }
void emitStaticLoc(HTS& env, int32_t locId, const StringData* name) { if (curFunc(env)->isPseudoMain()) PUNT(StaticLoc); auto const ldPMExit = makePseudoMainExit(env); auto const box = curFunc(env)->isClosureBody() ? gen(env, ClosureStaticLocInit, cns(env, name), fp(env), cns(env, Type::Uninit)) : gen(env, LdStaticLocCached, StaticLocName { curFunc(env), name }); auto const res = cond( env, 0, [&] (Block* taken) { gen(env, CheckStaticLocInit, taken, box); }, [&] { // Next: the static local is already initialized return cns(env, true); }, [&] { // Taken: need to initialize the static local /* * Even though this path is "cold", we're not marking it * unlikely because the size of the instructions this will * generate is about 10 bytes, which is not much larger than the * 5 byte jump to acold would be. * * One note about StaticLoc: we're literally always going to * generate a fallthrough trace here that is cold (the code that * initializes the static local). TODO(#2894612). */ gen(env, StaticLocInitCached, box, cns(env, Type::InitNull)); return cns(env, false); }); gen(env, IncRef, box); auto const oldValue = ldLoc(env, locId, ldPMExit, DataTypeGeneric); stLocRaw(env, locId, fp(env), box); gen(env, DecRef, oldValue); push(env, res); }
/* * Attempts to begin inlining, and returns whether or not it successed. * * When doing gen-time inlining, we set up a series of IR instructions * that looks like this: * * fp0 = DefFP * sp = DefSP<offset> * * // ... normal stuff happens ... * * // FPI region: * SpillFrame sp, ... * // ... probably some StStks due to argument expressions * fp2 = DefInlineFP<func,retBC,retSP,off> sp * * // ... callee body ... * * InlineReturn fp2 * * In DCE we attempt to remove the InlineReturn and DefInlineFP instructions if * they aren't needed. */ bool beginInlining(IRGS& env, unsigned numParams, const Func* target, Offset returnBcOffset) { auto const& fpiStack = env.irb->fpiStack(); assertx(!fpiStack.empty() && "Inlining does not support calls with the FPush* in a different Tracelet"); assertx(returnBcOffset >= 0 && "returnBcOffset before beginning of caller"); assertx(curFunc(env)->base() + returnBcOffset < curFunc(env)->past() && "returnBcOffset past end of caller"); FTRACE(1, "[[[ begin inlining: {}\n", target->fullName()->data()); SSATmp** params = (SSATmp**)alloca(sizeof(SSATmp*) * numParams); for (unsigned i = 0; i < numParams; ++i) { params[numParams - i - 1] = popF(env); } auto const prevSP = fpiStack.front().returnSP; auto const prevSPOff = fpiStack.front().returnSPOff; spillStack(env); auto const calleeSP = sp(env); always_assert_flog( prevSP == calleeSP, "FPI stack pointer and callee stack pointer didn't match in beginInlining" ); auto const& info = fpiStack.front(); always_assert(!isFPushCuf(info.fpushOpc) && !info.interp); auto ctx = [&] { if (info.ctx || isFPushFunc(info.fpushOpc)) { return info.ctx; } constexpr int32_t adjust = offsetof(ActRec, m_r) - offsetof(ActRec, m_this); IRSPOffset ctxOff{invSPOff(env) - info.returnSPOff - adjust}; return gen(env, LdStk, TCtx, IRSPOffsetData{ctxOff}, sp(env)); }(); DefInlineFPData data; data.target = target; data.retBCOff = returnBcOffset; data.fromFPushCtor = isFPushCtor(info.fpushOpc); data.ctx = ctx; data.retSPOff = prevSPOff; data.spOffset = offsetFromIRSP(env, BCSPOffset{0}); // Push state and update the marker before emitting any instructions so // they're all given markers in the callee. auto const key = SrcKey { target, target->getEntryForNumArgs(numParams), false }; env.bcStateStack.emplace_back(key); env.inlineLevel++; updateMarker(env); auto const calleeFP = gen(env, DefInlineFP, data, calleeSP, fp(env)); for (unsigned i = 0; i < numParams; ++i) { stLocRaw(env, i, calleeFP, params[i]); } const bool hasVariadicArg = target->hasVariadicCaptureParam(); for (unsigned i = numParams; i < target->numLocals() - hasVariadicArg; ++i) { /* * Here we need to be generating hopefully-dead stores to initialize * non-parameter locals to KindOfUninit in case we have to leave the trace. */ stLocRaw(env, i, calleeFP, cns(env, TUninit)); } if (hasVariadicArg) { auto argNum = target->numLocals() - 1; always_assert(numParams <= argNum); stLocRaw(env, argNum, calleeFP, cns(env, staticEmptyArray())); } return true; }
/* * Attempts to begin inlining, and returns whether or not it successed. * * When doing gen-time inlining, we set up a series of IR instructions * that looks like this: * * fp0 = DefFP * sp = DefSP<offset> * * // ... normal stuff happens ... * * // FPI region: * SpillFrame sp, ... * // ... probably some StStks due to argument expressions * fp2 = DefInlineFP<func,retBC,retSP,off> sp * * // ... callee body ... * * InlineReturn fp2 * * In DCE we attempt to remove the InlineReturn and DefInlineFP instructions if * they aren't needed. */ bool beginInlining(IRGS& env, unsigned numParams, const Func* target, Offset returnBcOffset) { assertx(!env.fpiStack.empty() && "Inlining does not support calls with the FPush* in a different Tracelet"); assertx(returnBcOffset >= 0 && "returnBcOffset before beginning of caller"); assertx(curFunc(env)->base() + returnBcOffset < curFunc(env)->past() && "returnBcOffset past end of caller"); FTRACE(1, "[[[ begin inlining: {}\n", target->fullName()->data()); SSATmp* params[numParams]; for (unsigned i = 0; i < numParams; ++i) { params[numParams - i - 1] = popF(env); } auto const prevSP = env.fpiStack.top().returnSP; auto const prevSPOff = env.fpiStack.top().returnSPOff; spillStack(env); auto const calleeSP = sp(env); always_assert_flog( prevSP == calleeSP, "FPI stack pointer and callee stack pointer didn't match in beginInlining" ); // This can only happen if the code is unreachable, in which case // the FPush* can punt if it gets a TBottom. if (env.fpiStack.top().spillFrame == nullptr) return false; auto const sframe = env.fpiStack.top().spillFrame; DefInlineFPData data; data.target = target; data.retBCOff = returnBcOffset; data.fromFPushCtor = sframe->extra<ActRecInfo>()->fromFPushCtor; data.ctx = sframe->src(2); data.retSPOff = prevSPOff; data.spOffset = offsetFromIRSP(env, BCSPOffset{0}); // Push state and update the marker before emitting any instructions so // they're all given markers in the callee. auto const key = SrcKey { target, target->getEntryForNumArgs(numParams), false }; env.bcStateStack.emplace_back(key); env.inlineLevel++; updateMarker(env); auto const calleeFP = gen(env, DefInlineFP, data, calleeSP, fp(env)); for (unsigned i = 0; i < numParams; ++i) { stLocRaw(env, i, calleeFP, params[i]); } const bool hasVariadicArg = target->hasVariadicCaptureParam(); for (unsigned i = numParams; i < target->numLocals() - hasVariadicArg; ++i) { /* * Here we need to be generating hopefully-dead stores to initialize * non-parameter locals to KindOfUninit in case we have to leave the trace. */ stLocRaw(env, i, calleeFP, cns(env, TUninit)); } if (hasVariadicArg) { auto argNum = target->numLocals() - 1; always_assert(numParams <= argNum); stLocRaw(env, argNum, calleeFP, cns(env, staticEmptyArray())); } env.fpiActiveStack.push(std::make_pair(env.fpiStack.top().returnSP, env.fpiStack.top().returnSPOff)); env.fpiStack.pop(); return true; }
/* * When doing gen-time inlining, we set up a series of IR instructions * that looks like this: * * fp0 = DefFP * sp0 = DefSP<offset> * * // ... normal stuff happens ... * // sp_pre = some SpillStack, or maybe the DefSP * * // FPI region: * sp1 = SpillStack sp_pre, ... * sp2 = SpillFrame sp1, ... * // ... possibly more spillstacks due to argument expressions * sp3 = SpillStack sp2, -argCount * fp2 = DefInlineFP<func,retBC,retSP,off> sp2 sp1 * sp4 = ReDefSP<spOffset,spansCall> sp1 fp2 * * // ... callee body ... * * = InlineReturn fp2 * * [ sp5 = ResetSP<spOffset> fp0 ] * * The rest of the code then depends on sp5, and not any of the StkPtr * tree going through the callee body. The sp5 tmp has the same view * of the stack as sp1 did, which represents what the stack looks like * before the return address is pushed but after the activation record * is popped. * * In DCE we attempt to remove the SpillFrame, InlineReturn, and * DefInlineFP instructions if they aren't needed. * * ReDefSP takes sp1, the stack pointer from before the inlined frame. * This SSATmp may be used for determining stack types in the * simplifier, or stack values if the inlined body doesn't contain a * call---these instructions both take an extradata `spansCall' which * is true iff a Call occured anywhere between the the definition of * its first argument and itself. */ void beginInlining(HTS& env, unsigned numParams, const Func* target, Offset returnBcOffset) { assert(!env.fpiStack.empty() && "Inlining does not support calls with the FPush* in a different Tracelet"); assert(returnBcOffset >= 0 && "returnBcOffset before beginning of caller"); assert(curFunc(env)->base() + returnBcOffset < curFunc(env)->past() && "returnBcOffset past end of caller"); FTRACE(1, "[[[ begin inlining: {}\n", target->fullName()->data()); SSATmp* params[numParams]; for (unsigned i = 0; i < numParams; ++i) { params[numParams - i - 1] = popF(env); } auto const prevSP = env.fpiStack.top().returnSP; auto const prevSPOff = env.fpiStack.top().returnSPOff; spillStack(env); auto const calleeSP = sp(env); always_assert_flog( env.fpiStack.top().spillFrame != nullptr, "Couldn't find SpillFrame for inlined call on sp {}." " Was the FPush instruction interpreted?\n{}", *calleeSP->inst(), env.irb->unit() ); auto const sframe = env.fpiStack.top().spillFrame; DefInlineFPData data; data.target = target; data.retBCOff = returnBcOffset; data.fromFPushCtor = sframe->extra<ActRecInfo>()->isFromFPushCtor(); data.ctx = sframe->src(2); data.retSPOff = prevSPOff; data.spOffset = offsetFromSP(env, 0); // Push state and update the marker before emitting any instructions so // they're all given markers in the callee. auto const key = SrcKey { target, target->getEntryForNumArgs(numParams), false }; env.bcStateStack.emplace_back(key); updateMarker(env); auto const calleeFP = gen(env, DefInlineFP, data, calleeSP, prevSP, fp(env)); gen(env, ReDefSP, StackOffset{target->numLocals()}, fp(env)); for (unsigned i = 0; i < numParams; ++i) { stLocRaw(env, i, calleeFP, params[i]); } for (unsigned i = numParams; i < target->numLocals(); ++i) { /* * Here we need to be generating hopefully-dead stores to initialize * non-parameter locals to KindOfUninit in case we have to leave the trace. */ stLocRaw(env, i, calleeFP, cns(env, Type::Uninit)); } env.fpiActiveStack.push(std::make_pair(env.fpiStack.top().returnSP, env.fpiStack.top().returnSPOff)); env.fpiStack.pop(); }
void emitPushL(HTS& env, int32_t id) { assertTypeLocal(env, id, Type::InitCell); // bytecode invariant auto* locVal = ldLoc(env, id, makeExit(env), DataTypeGeneric); push(env, locVal); stLocRaw(env, id, fp(env), cns(env, Type::Uninit)); }
void emitUnsetL(HTS& env, int32_t id) { auto const prev = ldLoc(env, id, makeExit(env), DataTypeCountness); stLocRaw(env, id, fp(env), cns(env, Type::Uninit)); gen(env, DecRef, prev); }