void emitCreateCont(IRGS& env) {
auto const resumeOffset = nextBcOff(env);
assertx(!resumed(env));
assertx(curFunc(env)->isGenerator());
if (curFunc(env)->isAsyncGenerator()) PUNT(CreateCont-AsyncGenerator);
// Create the Generator object. CreateCont takes care of copying local
// variables and iterators.
auto const func = curFunc(env);
auto const resumeSk = SrcKey(func, resumeOffset, true);
auto const bind_data = LdBindAddrData { resumeSk, invSPOff(env) + 1 };
auto const resumeAddr = gen(env, LdBindAddr, bind_data);
auto const cont =
gen(env,
CreateCont,
fp(env),
cns(env, func->numSlotsInFrame()),
resumeAddr,
cns(env, resumeOffset));
// The suspend hook will decref the newly created generator if it throws.
auto const contAR =
gen(env,
LdContActRec,
IsAsyncData(curFunc(env)->isAsync()),
cont);
suspendHookE(env, fp(env), contAR, cont);
// Grab caller info from ActRec, free ActRec, store the return value
// and return control to the caller.
gen(env, StRetVal, fp(env), cont);
auto const ret_data = RetCtrlData { offsetToReturnSlot(env), false };
gen(env, RetCtrl, ret_data, sp(env), fp(env));
}
void tearDownEagerAsyncFrame(ActRec*& fp, Stack& stack, PC& pc, ObjectData* e) {
auto const func = fp->func();
auto const prevFp = fp->sfp();
auto const soff = fp->m_soff;
assert(!fp->resumed());
assert(func->isAsyncFunction());
assert(*reinterpret_cast<const Op*>(pc) != OpRetC);
FTRACE(1, "tearDownAsyncFrame: {} ({})\n fp {} prevFp {}\n",
func->fullName()->data(),
func->unit()->filepath()->data(),
implicit_cast<void*>(fp),
implicit_cast<void*>(prevFp));
try {
frame_free_locals_unwind(fp, func->numLocals());
} catch (...) {}
stack.ndiscard(func->numSlotsInFrame());
stack.ret();
assert(stack.topTV() == &fp->m_r);
tvWriteObject(c_StaticWaitHandle::CreateFailed(e), &fp->m_r);
e->decRefCount();
if (UNLIKELY(!prevFp)) {
pc = 0;
return;
}
assert(stack.isValidAddress(reinterpret_cast<uintptr_t>(prevFp)) ||
prevFp->resumed());
auto const prevOff = soff + prevFp->func()->base();
pc = prevFp->func()->unit()->at(prevOff);
fp = prevFp;
}
void emitCreateCont(HTS& env) {
auto const resumeOffset = nextBcOff(env);
assert(!resumed(env));
assert(curFunc(env)->isGenerator());
if (curFunc(env)->isAsyncGenerator()) PUNT(CreateCont-AsyncGenerator);
// Create the Generator object. CreateCont takes care of copying local
// variables and iterators.
auto const func = curFunc(env);
auto const resumeSk = SrcKey(func, resumeOffset, true);
auto const resumeAddr = gen(env, LdBindAddr, LdBindAddrData(resumeSk));
auto const cont =
gen(env,
CreateCont,
fp(env),
cns(env, func->numSlotsInFrame()),
resumeAddr,
cns(env, resumeOffset));
// The suspend hook will decref the newly created generator if it throws.
auto const contAR = gen(env, LdContActRec, cont);
suspendHookE(env, fp(env), contAR);
// Grab caller info from ActRec, free ActRec, store the return value
// and return control to the caller.
gen(env, StRetVal, fp(env), cont);
auto const retAddr = gen(env, LdRetAddr, fp(env));
auto const stack = gen(env, RetAdjustStack, fp(env));
auto const frame = gen(env, FreeActRec, fp(env));
gen(env, RetCtrl, RetCtrlData(false), stack, frame, retAddr);
}
void cgCheckSurpriseAndStack(IRLS& env, const IRInstruction* inst) {
auto const fp = srcLoc(env, inst, 0).reg();
auto const extra = inst->extra<CheckSurpriseAndStack>();
auto const func = extra->func;
auto const off = func->getEntryForNumArgs(extra->argc) - func->base();
auto const fixup = Fixup(off, func->numSlotsInFrame());
auto& v = vmain(env);
auto const sf = v.makeReg();
auto const needed_top = v.makeReg();
v << lea{fp[-cellsToBytes(func->maxStackCells())], needed_top};
v << cmpqm{needed_top, rvmtl()[rds::kSurpriseFlagsOff], sf};
unlikelyIfThen(v, vcold(env), CC_AE, sf, [&] (Vout& v) {
auto const stub = tc::ustubs().functionSurprisedOrStackOverflow;
auto const done = v.makeBlock();
v << vinvoke{CallSpec::stub(stub), v.makeVcallArgs({}), v.makeTuple({}),
{done, label(env, inst->taken())}, fixup };
v = done;
});
}
void tearDownFrame(ActRec*& fp, Stack& stack, PC& pc, Offset& faultOffset) {
auto const func = fp->m_func;
auto const curOp = *reinterpret_cast<const Op*>(pc);
auto const unwindingGeneratorFrame = func->isGenerator();
auto const unwindingReturningFrame = curOp == OpRetC || curOp == OpRetV;
auto const prevFp = fp->arGetSfp();
auto const soff = fp->m_soff;
FTRACE(1, "tearDownFrame: {} ({})\n fp {} prevFp {}\n",
func->fullName()->data(),
func->unit()->filepath()->data(),
implicit_cast<void*>(fp),
implicit_cast<void*>(prevFp));
// When throwing from a constructor, we normally want to avoid running the
// destructor on an object that hasn't been fully constructed yet. But if
// we're unwinding through the constructor's RetC, the constructor has
// logically finished and we're unwinding for some internal reason (timeout
// or user profiler, most likely). More importantly, fp->m_this may have
// already been destructed and/or overwritten due to sharing space with
// fp->m_r.
if (!unwindingReturningFrame && fp->isFromFPushCtor() && fp->hasThis()) {
fp->getThis()->setNoDestruct();
}
// A generator's locals don't live on this stack.
if (LIKELY(!unwindingGeneratorFrame)) {
/*
* If we're unwinding through a frame that's returning, it's only
* possible that its locals have already been decref'd.
*
* Here's why:
*
* - If a destructor for any of these things throws a php
* exception, it's swallowed at the dtor boundary and we keep
* running php.
*
* - If the destructor for any of these things throws a fatal,
* it's swallowed, and we set surprise flags to throw a fatal
* from now on.
*
* - If the second case happened and we have to run another
* destructor, its enter hook will throw, but it will be
* swallowed again.
*
* - Finally, the exit hook for the returning function can
* throw, but this happens last so everything is destructed.
*
*/
if (!unwindingReturningFrame) {
try {
// Note that we must convert locals and the $this to
// uninit/zero during unwind. This is because a backtrace
// from another destructing object during this unwind may try
// to read them.
frame_free_locals_unwind(fp, func->numLocals());
} catch (...) {}
}
stack.ndiscard(func->numSlotsInFrame());
stack.discardAR();
} else {
// The generator's locals will be cleaned up when the Continuation
// object is destroyed. But we are leaving the generator function
// now, so signal that to anyone who cares.
try {
EventHook::FunctionExit(fp);
} catch (...) {} // As above, don't let new exceptions out of unwind.
}
/*
* At the final ActRec in this nesting level. We don't need to set
* pc and fp since we're about to re-throw the exception. And we
* don't want to dereference prefFp since we just popped it.
*/
if (prevFp == fp) return;
assert(stack.isValidAddress(reinterpret_cast<uintptr_t>(prevFp)) ||
prevFp->m_func->isGenerator());
auto const prevOff = soff + prevFp->m_func->base();
pc = prevFp->m_func->unit()->at(prevOff);
fp = prevFp;
faultOffset = prevOff;
}
bool RegionFormer::tryInline(uint32_t& instrSize) {
assertx(m_inst.source == m_sk);
assertx(m_inst.func() == curFunc());
assertx(m_sk.resumed() == resumed());
instrSize = 0;
if (!m_inl.canInlineAt(m_inst.source, m_inst.funcd, *m_region)) {
return false;
}
auto refuse = [this](const std::string& str) {
FTRACE(2, "selectTracelet not inlining {}: {}\n",
m_inst.toString(), str);
return false;
};
auto callee = m_inst.funcd;
// Make sure the FPushOp wasn't interpreted.
if (m_irgs.fpiStack.empty()) {
return refuse("fpistack empty; fpush was in a different region");
}
auto spillFrame = m_irgs.fpiStack.top().spillFrame;
if (!spillFrame) {
return refuse("couldn't find SpillFrame for FPushOp");
}
auto numArgs = m_inst.imm[0].u_IVA;
auto numParams = callee->numParams();
// Set up the region context, mapping stack slots in the caller to locals in
// the callee.
RegionContext ctx;
ctx.func = callee;
ctx.bcOffset = callee->getEntryForNumArgs(numArgs);
ctx.spOffset = FPInvOffset{safe_cast<int32_t>(callee->numSlotsInFrame())};
ctx.resumed = false;
for (int i = 0; i < numArgs; ++i) {
auto type = irgen::publicTopType(m_irgs, BCSPOffset{i});
uint32_t paramIdx = numArgs - 1 - i;
ctx.liveTypes.push_back({RegionDesc::Location::Local{paramIdx}, type});
}
for (unsigned i = numArgs; i < numParams; ++i) {
// These locals will be populated by DV init funclets but they'll start
// out as Uninit.
ctx.liveTypes.push_back({RegionDesc::Location::Local{i}, TUninit});
}
FTRACE(1, "selectTracelet analyzing callee {} with context:\n{}",
callee->fullName()->data(), show(ctx));
auto region = selectTracelet(ctx, m_profiling, false /* noinline */);
if (!region) {
return refuse("failed to select region in callee");
}
instrSize = region->instrSize();
auto newInstrSize = instrSize + m_numBCInstrs + m_pendingInlinedInstrs;
if (newInstrSize > RuntimeOption::EvalJitMaxRegionInstrs) {
return refuse("new region would be too large");
}
if (!m_inl.shouldInline(callee, *region)) {
return refuse("shouldIRInline failed");
}
return true;
}
void tearDownFrame(ActRec*& fp, Stack& stack, PC& pc) {
auto const func = fp->func();
auto const curOp = *reinterpret_cast<const Op*>(pc);
auto const prevFp = fp->sfp();
auto const soff = fp->m_soff;
FTRACE(1, "tearDownFrame: {} ({})\n fp {} prevFp {}\n",
func->fullName()->data(),
func->unit()->filepath()->data(),
implicit_cast<void*>(fp),
implicit_cast<void*>(prevFp));
// When throwing from a constructor, we normally want to avoid running the
// destructor on an object that hasn't been fully constructed yet. But if
// we're unwinding through the constructor's RetC, the constructor has
// logically finished and we're unwinding for some internal reason (timeout
// or user profiler, most likely). More importantly, fp->m_this may have
// already been destructed and/or overwritten due to sharing space with
// fp->m_r.
if (fp->isFromFPushCtor() && fp->hasThis() && curOp != OpRetC) {
fp->getThis()->setNoDestruct();
}
/*
* It is possible that locals have already been decref'd.
*
* Here's why:
*
* - If a destructor for any of these things throws a php
* exception, it's swallowed at the dtor boundary and we keep
* running php.
*
* - If the destructor for any of these things throws a fatal,
* it's swallowed, and we set surprise flags to throw a fatal
* from now on.
*
* - If the second case happened and we have to run another
* destructor, its enter hook will throw, but it will be
* swallowed again.
*
* - Finally, the exit hook for the returning function can
* throw, but this happens last so everything is destructed.
*
* - When that happens, exit hook sets localsDecRefd flag.
*/
if (!fp->localsDecRefd()) {
try {
// Note that we must convert locals and the $this to
// uninit/zero during unwind. This is because a backtrace
// from another destructing object during this unwind may try
// to read them.
frame_free_locals_unwind(fp, func->numLocals());
} catch (...) {}
}
if (LIKELY(!fp->resumed())) {
// Free ActRec.
stack.ndiscard(func->numSlotsInFrame());
stack.discardAR();
} else if (fp->func()->isAsyncFunction()) {
// Do nothing. AsyncFunctionWaitHandle will handle the exception.
} else if (fp->func()->isAsyncGenerator()) {
// Do nothing. AsyncGeneratorWaitHandle will handle the exception.
} else if (fp->func()->isNonAsyncGenerator()) {
// Mark the generator as finished.
frame_generator(fp)->finish();
} else {
not_reached();
}
/*
* At the final ActRec in this nesting level. We don't need to set
* pc and fp since we're about to re-throw the exception. And we
* don't want to dereference prefFp since we just popped it.
*/
if (!prevFp) return;
assert(stack.isValidAddress(reinterpret_cast<uintptr_t>(prevFp)) ||
prevFp->resumed());
auto const prevOff = soff + prevFp->func()->base();
pc = prevFp->func()->unit()->at(prevOff);
fp = prevFp;
}
/**
* Discard the current frame, assuming that a PHP exception given in
* phpException argument, or C++ exception (phpException == nullptr)
* is being thrown. Returns an exception to propagate, or nulltpr
* if the VM execution should be resumed.
*/
ObjectData* tearDownFrame(ActRec*& fp, Stack& stack, PC& pc,
ObjectData* phpException) {
auto const func = fp->func();
auto const curOp = peek_op(pc);
auto const prevFp = fp->sfp();
auto const soff = fp->m_soff;
ITRACE(1, "tearDownFrame: {} ({})\n",
func->fullName()->data(),
func->unit()->filepath()->data());
ITRACE(1, " fp {} prevFp {}\n",
implicit_cast<void*>(fp),
implicit_cast<void*>(prevFp));
// When throwing from a constructor, we normally want to avoid running the
// destructor on an object that hasn't been fully constructed yet. But if
// we're unwinding through the constructor's RetC, the constructor has
// logically finished and we're unwinding for some internal reason (timeout
// or user profiler, most likely). More importantly, fp->m_this may have
// already been destructed and/or overwritten due to sharing space with
// fp->m_r.
if (curOp != OpRetC &&
fp->hasThis() &&
fp->getThis()->getVMClass()->getCtor() == func &&
fp->getThis()->getVMClass()->getDtor()) {
/*
* Looks like an FPushCtor call, but it could still have been called
* directly. Check the fpi region to be sure.
*/
Offset prevPc;
auto outer = g_context->getPrevVMState(fp, &prevPc);
if (outer) {
auto fe = outer->func()->findPrecedingFPI(prevPc);
if (fe && isFPushCtor(outer->func()->unit()->getOp(fe->m_fpushOff))) {
fp->getThis()->setNoDestruct();
}
}
}
auto const decRefLocals = [&] {
/*
* It is possible that locals have already been decref'd.
*
* Here's why:
*
* - If a destructor for any of these things throws a php
* exception, it's swallowed at the dtor boundary and we keep
* running php.
*
* - If the destructor for any of these things throws a fatal,
* it's swallowed, and we set surprise flags to throw a fatal
* from now on.
*
* - If the second case happened and we have to run another
* destructor, its enter hook will throw, but it will be
* swallowed again.
*
* - Finally, the exit hook for the returning function can
* throw, but this happens last so everything is destructed.
*
* - When that happens, exit hook sets localsDecRefd flag.
*/
if (!fp->localsDecRefd()) {
try {
// Note that we must convert locals and the $this to
// uninit/zero during unwind. This is because a backtrace
// from another destructing object during this unwind may try
// to read them.
frame_free_locals_unwind(fp, func->numLocals(), phpException);
} catch (...) {}
}
};
if (LIKELY(!fp->resumed())) {
decRefLocals();
if (UNLIKELY(func->isAsyncFunction()) &&
phpException &&
!fp->isFCallAwait()) {
// If in an eagerly executed async function, wrap the user exception
// into a failed StaticWaitHandle and return it to the caller.
auto const waitHandle = c_StaticWaitHandle::CreateFailed(phpException);
phpException = nullptr;
stack.ndiscard(func->numSlotsInFrame());
stack.ret();
assert(stack.topTV() == &fp->m_r);
cellCopy(make_tv<KindOfObject>(waitHandle), fp->m_r);
} else {
// Free ActRec.
stack.ndiscard(func->numSlotsInFrame());
stack.discardAR();
}
} else if (func->isAsyncFunction()) {
auto const waitHandle = frame_afwh(fp);
if (phpException) {
// Handle exception thrown by async function.
decRefLocals();
waitHandle->fail(phpException);
phpException = nullptr;
} else if (waitHandle->isRunning()) {
// Let the C++ exception propagate. If the current frame represents async
// function that is running, mark it as abruptly interrupted. Some opcodes
// like Await may change state of the async function just before exit hook
// decides to throw C++ exception.
decRefLocals();
waitHandle->failCpp();
}
} else if (func->isAsyncGenerator()) {
auto const gen = frame_async_generator(fp);
if (phpException) {
// Handle exception thrown by async generator.
decRefLocals();
auto eagerResult = gen->fail(phpException);
phpException = nullptr;
if (eagerResult) {
stack.pushObjectNoRc(eagerResult);
}
} else if (gen->isEagerlyExecuted() || gen->getWaitHandle()->isRunning()) {
// Fail the async generator and let the C++ exception propagate.
decRefLocals();
gen->failCpp();
}
} else if (func->isNonAsyncGenerator()) {
// Mark the generator as finished.
decRefLocals();
frame_generator(fp)->fail();
} else {
not_reached();
}
/*
* At the final ActRec in this nesting level.
*/
if (UNLIKELY(!prevFp)) {
pc = nullptr;
fp = nullptr;
return phpException;
}
assert(stack.isValidAddress(reinterpret_cast<uintptr_t>(prevFp)) ||
prevFp->resumed());
auto const prevOff = soff + prevFp->func()->base();
pc = prevFp->func()->unit()->at(prevOff);
fp = prevFp;
return phpException;
}
UnwindAction tearDownFrame(ActRec*& fp, Stack& stack, PC& pc,
const Fault& fault) {
auto const func = fp->func();
auto const curOp = *reinterpret_cast<const Op*>(pc);
auto const prevFp = fp->sfp();
auto const soff = fp->m_soff;
FTRACE(1, "tearDownFrame: {} ({})\n fp {} prevFp {}\n",
func->fullName()->data(),
func->unit()->filepath()->data(),
implicit_cast<void*>(fp),
implicit_cast<void*>(prevFp));
// When throwing from a constructor, we normally want to avoid running the
// destructor on an object that hasn't been fully constructed yet. But if
// we're unwinding through the constructor's RetC, the constructor has
// logically finished and we're unwinding for some internal reason (timeout
// or user profiler, most likely). More importantly, fp->m_this may have
// already been destructed and/or overwritten due to sharing space with
// fp->m_r.
if (fp->isFromFPushCtor() && fp->hasThis() && curOp != OpRetC) {
fp->getThis()->setNoDestruct();
}
/*
* It is possible that locals have already been decref'd.
*
* Here's why:
*
* - If a destructor for any of these things throws a php
* exception, it's swallowed at the dtor boundary and we keep
* running php.
*
* - If the destructor for any of these things throws a fatal,
* it's swallowed, and we set surprise flags to throw a fatal
* from now on.
*
* - If the second case happened and we have to run another
* destructor, its enter hook will throw, but it will be
* swallowed again.
*
* - Finally, the exit hook for the returning function can
* throw, but this happens last so everything is destructed.
*
* - When that happens, exit hook sets localsDecRefd flag.
*/
if (!fp->localsDecRefd()) {
try {
// Note that we must convert locals and the $this to
// uninit/zero during unwind. This is because a backtrace
// from another destructing object during this unwind may try
// to read them.
frame_free_locals_unwind(fp, func->numLocals(), fault);
} catch (...) {}
}
auto action = UnwindAction::Propagate;
if (LIKELY(!fp->resumed())) {
if (UNLIKELY(func->isAsyncFunction()) &&
fault.m_faultType == Fault::Type::UserException) {
// If in an eagerly executed async function, wrap the user exception
// into a failed StaticWaitHandle and return it to the caller.
auto const e = fault.m_userException;
stack.ndiscard(func->numSlotsInFrame());
stack.ret();
assert(stack.topTV() == &fp->m_r);
tvWriteObject(c_StaticWaitHandle::CreateFailed(e), &fp->m_r);
e->decRefCount();
action = UnwindAction::ResumeVM;
} else {
// Free ActRec.
stack.ndiscard(func->numSlotsInFrame());
stack.discardAR();
}
} else if (func->isAsyncFunction()) {
auto const waitHandle = frame_afwh(fp);
if (fault.m_faultType == Fault::Type::UserException) {
// Handle exception thrown by async function.
waitHandle->fail(fault.m_userException);
action = UnwindAction::ResumeVM;
} else {
// Fail the async function and let the C++ exception propagate.
waitHandle->fail(AsioSession::Get()->getAbruptInterruptException());
}
} else if (func->isAsyncGenerator()) {
// Do nothing. AsyncGeneratorWaitHandle will handle the exception.
} else if (func->isNonAsyncGenerator()) {
// Mark the generator as finished.
frame_generator(fp)->finish();
} else {
not_reached();
}
/*
* At the final ActRec in this nesting level.
*/
if (UNLIKELY(!prevFp)) {
pc = nullptr;
fp = nullptr;
return action;
}
assert(stack.isValidAddress(reinterpret_cast<uintptr_t>(prevFp)) ||
prevFp->resumed());
auto const prevOff = soff + prevFp->func()->base();
pc = prevFp->func()->unit()->at(prevOff);
fp = prevFp;
return action;
}
void tearDownFrame(ActRec*& fp, Stack& stack, PC& pc, Offset& faultOffset) {
auto const func = fp->m_func;
auto const curOp = *reinterpret_cast<const Op*>(pc);
auto const unwindingGeneratorFrame = func->isGenerator();
auto const unwindingReturningFrame = curOp == OpRetC || curOp == OpRetV;
auto const prevFp = fp->arGetSfp();
FTRACE(1, "tearDownFrame: {} ({})\n fp {} prevFp {}\n",
func->fullName()->data(),
func->unit()->filepath()->data(),
implicit_cast<void*>(fp),
implicit_cast<void*>(prevFp));
if (fp->isFromFPushCtor() && fp->hasThis()) {
fp->getThis()->setNoDestruct();
}
// A generator's locals don't live on this stack.
if (LIKELY(!unwindingGeneratorFrame)) {
/*
* If we're unwinding through a frame that's returning, it's only
* possible that its locals have already been decref'd.
*
* Here's why:
*
* - If a destructor for any of these things throws a php
* exception, it's swallowed at the dtor boundary and we keep
* running php.
*
* - If the destructor for any of these things throws a fatal,
* it's swallowed, and we set surprise flags to throw a fatal
* from now on.
*
* - If the second case happened and we have to run another
* destructor, its enter hook will throw, but it will be
* swallowed again.
*
* - Finally, the exit hook for the returning function can
* throw, but this happens last so everything is destructed.
*
*/
if (!unwindingReturningFrame) {
try {
// Note that we must convert locals and the $this to
// uninit/zero during unwind. This is because a backtrace
// from another destructing object during this unwind may try
// to read them.
frame_free_locals_unwind(fp, func->numLocals());
} catch (...) {}
}
stack.ndiscard(func->numSlotsInFrame());
stack.discardAR();
}
assert(stack.isValidAddress(reinterpret_cast<uintptr_t>(prevFp)) ||
prevFp->m_func->isGenerator());
auto const prevOff = fp->m_soff + prevFp->m_func->base();
pc = prevFp->m_func->unit()->at(prevOff);
fp = prevFp;
faultOffset = prevOff;
}
