void emitContEnter(HTS& env) {
auto const returnOffset = nextBcOff(env);
assert(curClass(env));
assert(curClass(env)->classof(c_AsyncGenerator::classof()) ||
curClass(env)->classof(c_Generator::classof()));
assert(curFunc(env)->contains(returnOffset));
// Load generator's FP and resume address.
auto const genObj = ldThis(env);
auto const genFp = gen(env, LdContActRec, genObj);
auto resumeAddr = gen(env, LdContResumeAddr, genObj);
// Make sure function enter hook is called if needed.
auto const exitSlow = makeExitSlow(env);
gen(env, CheckSurpriseFlags, exitSlow);
// Exit to interpreter if resume address is not known.
resumeAddr = gen(env, CheckNonNull, exitSlow, resumeAddr);
// Sync stack.
auto const stack = spillStack(env);
// Enter generator.
auto returnBcOffset = returnOffset - curFunc(env)->base();
gen(env, ContEnter, stack, fp(env), genFp, resumeAddr,
cns(env, returnBcOffset));
}
void emitCreateCont(IRGS& env) {
auto const resumeOffset = nextBcOff(env);
assertx(!resumed(env));
assertx(curFunc(env)->isGenerator());
if (curFunc(env)->isAsyncGenerator()) PUNT(CreateCont-AsyncGenerator);
// Create the Generator object. CreateCont takes care of copying local
// variables and iterators.
auto const func = curFunc(env);
auto const resumeSk = SrcKey(func, resumeOffset, true);
auto const bind_data = LdBindAddrData { resumeSk, invSPOff(env) + 1 };
auto const resumeAddr = gen(env, LdBindAddr, bind_data);
auto const cont =
gen(env,
CreateCont,
fp(env),
cns(env, func->numSlotsInFrame()),
resumeAddr,
cns(env, resumeOffset));
// The suspend hook will decref the newly created generator if it throws.
auto const contAR =
gen(env,
LdContActRec,
IsAsyncData(curFunc(env)->isAsync()),
cont);
suspendHookE(env, fp(env), contAR, cont);
// Grab caller info from ActRec, free ActRec, store the return value
// and return control to the caller.
gen(env, StRetVal, fp(env), cont);
auto const ret_data = RetCtrlData { offsetToReturnSlot(env), false };
gen(env, RetCtrl, ret_data, sp(env), fp(env));
}
void emitStaticLocInit(HTS& env, int32_t locId, const StringData* name) {
if (curFunc(env)->isPseudoMain()) PUNT(StaticLocInit);
auto const ldPMExit = makePseudoMainExit(env);
auto const value = popC(env);
// Closures and generators from closures don't satisfy the "one static per
// source location" rule that the inline fastpath requires
auto const box = [&]{
if (curFunc(env)->isClosureBody()) {
return gen(env, ClosureStaticLocInit, cns(env, name), fp(env), value);
}
auto const cachedBox =
gen(env, LdStaticLocCached, StaticLocName { curFunc(env), name });
ifThen(
env,
[&] (Block* taken) {
gen(env, CheckStaticLocInit, taken, cachedBox);
},
[&] {
hint(env, Block::Hint::Unlikely);
gen(env, StaticLocInitCached, cachedBox, value);
}
);
return cachedBox;
}();
gen(env, IncRef, box);
auto const oldValue = ldLoc(env, locId, ldPMExit, DataTypeSpecific);
stLocRaw(env, locId, fp(env), box);
gen(env, DecRef, oldValue);
// We don't need to decref value---it's a bytecode invariant that
// our Cell was not ref-counted.
}
void emitCreateCont(HTS& env) {
auto const resumeOffset = nextBcOff(env);
assert(!resumed(env));
assert(curFunc(env)->isGenerator());
if (curFunc(env)->isAsyncGenerator()) PUNT(CreateCont-AsyncGenerator);
// Create the Generator object. CreateCont takes care of copying local
// variables and iterators.
auto const func = curFunc(env);
auto const resumeSk = SrcKey(func, resumeOffset, true);
auto const resumeAddr = gen(env, LdBindAddr, LdBindAddrData(resumeSk));
auto const cont =
gen(env,
CreateCont,
fp(env),
cns(env, func->numSlotsInFrame()),
resumeAddr,
cns(env, resumeOffset));
// The suspend hook will decref the newly created generator if it throws.
auto const contAR = gen(env, LdContActRec, cont);
suspendHookE(env, fp(env), contAR);
// Grab caller info from ActRec, free ActRec, store the return value
// and return control to the caller.
gen(env, StRetVal, fp(env), cont);
auto const retAddr = gen(env, LdRetAddr, fp(env));
auto const stack = gen(env, RetAdjustStack, fp(env));
auto const frame = gen(env, FreeActRec, fp(env));
gen(env, RetCtrl, RetCtrlData(false), stack, frame, retAddr);
}
void endInlinedCommon(IRGS& env) {
assertx(!curFunc(env)->isPseudoMain());
assertx(!resumed(env));
decRefLocalsInline(env);
decRefThis(env);
gen(env, InlineReturn, fp(env));
// Return to the caller function. Careful between here and the
// updateMarker() below, where the caller state isn't entirely set up.
env.inlineLevel--;
env.bcStateStack.pop_back();
always_assert(env.bcStateStack.size() > 0);
updateMarker(env);
/*
* After the end of inlining, we are restoring to a previously defined stack
* that we know is entirely materialized (i.e. in memory), so stackDeficit
* needs to be slammed to zero.
*
* The push of the return value in the caller of this function is not yet
* materialized.
*/
assertx(env.irb->evalStack().empty());
env.irb->clearStackDeficit();
FTRACE(1, "]]] end inlining: {}\n", curFunc(env)->fullName()->data());
}
void emitContEnter(IRGS& env) {
auto const returnOffset = nextBcOff(env);
assertx(curClass(env));
assertx(curClass(env)->classof(c_AsyncGenerator::classof()) ||
curClass(env)->classof(c_Generator::classof()));
assertx(curFunc(env)->contains(returnOffset));
auto isAsync = curClass(env)->classof(c_AsyncGenerator::classof());
// Load generator's FP and resume address.
auto const genObj = ldThis(env);
auto const genFp = gen(env, LdContActRec, IsAsyncData(isAsync), genObj);
auto resumeAddr = gen(env, LdContResumeAddr, IsAsyncData(isAsync), genObj);
// Make sure function enter hook is called if needed.
auto const exitSlow = makeExitSlow(env);
gen(env, CheckSurpriseFlags, exitSlow, fp(env));
// Exit to interpreter if resume address is not known.
resumeAddr = gen(env, CheckNonNull, exitSlow, resumeAddr);
spillStack(env);
env.irb->exceptionStackBoundary();
auto returnBcOffset = returnOffset - curFunc(env)->base();
gen(
env,
ContEnter,
ContEnterData { offsetFromIRSP(env, BCSPOffset{0}), returnBcOffset },
sp(env),
fp(env),
genFp,
resumeAddr
);
}
void CodeGenerator::cgReqBindJmp(IRInstruction* inst) {
emitBindJmp(
m_mainCode,
m_stubsCode,
SrcKey(curFunc(), inst->extra<ReqBindJmp>()->offset)
);
}
void CodeGenerator::cgInterpOneCommon(IRInstruction* inst) {
auto fpReg = x2a(curOpd(inst->src(0)).reg());
auto spReg = x2a(curOpd(inst->src(1)).reg());
auto pcOff = inst->extra<InterpOneData>()->bcOff;
auto opc = *(curFunc()->unit()->at(pcOff));
auto* interpOneHelper = interpOneEntryPoints[opc];
// This means push x30 (the link register) first, then x29. This mimics the
// x64 stack frame: return address higher in memory than saved FP.
m_as. Push (x30, x29);
// TODO(2966997): this really should be saving caller-save registers and
// basically doing everything else that cgCallHelper does. This only works
// now because no caller-saved registers are live.
m_as. Mov (rHostCallReg, reinterpret_cast<uint64_t>(interpOneHelper));
m_as. Mov (argReg(0), fpReg);
m_as. Mov (argReg(1), spReg);
m_as. Mov (argReg(2), pcOff);
// Note that sync points for HostCalls have to be recorded at the *start* of
// the instruction.
recordHostCallSyncPoint(m_as, m_as.frontier());
m_as. HostCall(3);
m_as. Pop (x29, x30);
}
void annotate(NormalizedInstruction* i) {
switch(i->op()) {
case OpFPushObjMethodD:
case OpFPushClsMethodD:
case OpFPushClsMethodF:
case OpFPushFuncD: {
// When we push predictable action records, we can use a simpler
// translation for their corresponding FCall.
SrcKey next(i->source);
next.advance(curUnit());
const StringData* className = NULL;
const StringData* funcName = NULL;
if (i->op() == OpFPushFuncD) {
funcName = curUnit()->lookupLitstrId(i->imm[1].u_SA);
} else if (i->op() == OpFPushObjMethodD) {
if (i->inputs[0]->valueType() != KindOfObject) break;
const Class* cls = i->inputs[0]->rtt.valueClass();
if (!cls) break;
funcName = curUnit()->lookupLitstrId(i->imm[1].u_SA);
className = cls->name();
} else if (i->op() == OpFPushClsMethodF) {
if (i->inputs[1]->rtt.valueString() == NULL ||
i->inputs[0]->valueType() != KindOfClass) {
break;
}
const Class* cls = i->inputs[0]->rtt.valueClass();
if (!cls) break;
funcName = i->inputs[1]->rtt.valueString();
className = cls->name();
} else {
ASSERT(i->op() == OpFPushClsMethodD);
funcName = curUnit()->lookupLitstrId(i->imm[1].u_SA);
className = curUnit()->lookupLitstrId(i->imm[2].u_SA);
}
ASSERT(funcName->isStatic());
const FPIEnt *fe = curFunc()->findFPI(next.m_offset);
ASSERT(fe);
recordActRecPush(i->source, curUnit(), fe, funcName, className,
i->op() == OpFPushClsMethodD ||
i->op() == OpFPushClsMethodF);
} break;
case OpFCall: {
CallRecord callRec;
if (mapGet(s_callDB, i->source, &callRec)) {
if (callRec.m_type == Function) {
i->funcd = callRec.m_func;
} else {
ASSERT(callRec.m_type == EncodedNameAndArgs);
i->funcName = callRec.m_encodedName;
}
} else {
i->funcName = NULL;
}
} break;
default: break;
}
}
void emitRetV(IRGS& env) {
assertx(!resumed(env));
assertx(!curFunc(env)->isResumable());
if (isInlining(env)) {
retFromInlined(env);
} else {
implRet(env);
}
}
void emitRetV(HTS& env) {
assert(!resumed(env));
assert(!curFunc(env)->isResumable());
if (isInlining(env)) {
retFromInlined(env, Type::BoxedInitCell);
} else {
implRet(env, Type::BoxedInitCell);
}
}
void emitRetC(IRGS& env) {
if (curFunc(env)->isAsyncGenerator()) PUNT(RetC-AsyncGenerator);
if (isInlining(env)) {
assertx(!resumed(env));
retFromInlined(env);
} else {
implRet(env);
}
}
void emitAwait(HTS& env, int32_t numIters) {
auto const resumeOffset = nextBcOff(env);
assert(curFunc(env)->isAsync());
if (curFunc(env)->isAsyncGenerator()) PUNT(Await-AsyncGenerator);
auto const exitSlow = makeExitSlow(env);
if (!topC(env)->isA(Type::Obj)) PUNT(Await-NonObject);
auto const child = popC(env);
gen(env, JmpZero, exitSlow, gen(env, IsWaitHandle, child));
// cns() would ODR-use these
auto const kSucceeded = c_WaitHandle::STATE_SUCCEEDED;
auto const kFailed = c_WaitHandle::STATE_FAILED;
auto const state = gen(env, LdWHState, child);
auto const failed = gen(env, EqInt, state, cns(env, kFailed));
gen(env, JmpNZero, exitSlow, failed);
env.irb->ifThenElse(
[&] (Block* taken) {
auto const succeeded = gen(env, EqInt, state, cns(env, kSucceeded));
gen(env, JmpNZero, taken, succeeded);
},
[&] { // Next: the wait handle is not finished, we need to suspend
if (resumed(env)) {
implAwaitR(env, child, resumeOffset);
} else {
implAwaitE(env, child, resumeOffset, numIters);
}
},
[&] { // Taken: retrieve the result from the wait handle
auto const res = gen(env, LdWHResult, child);
gen(env, IncRef, res);
gen(env, DecRef, child);
push(env, res);
}
);
}
void emitYieldK(IRGS& env) {
auto const resumeOffset = nextBcOff(env);
assertx(resumed(env));
assertx(curFunc(env)->isGenerator());
if (curFunc(env)->isAsyncGenerator()) PUNT(YieldK-AsyncGenerator);
yieldImpl(env, resumeOffset);
auto const newKey = popC(env);
auto const oldKey = gen(env, LdContArKey, TCell, fp(env));
gen(env, StContArKey, fp(env), newKey);
gen(env, DecRef, oldKey);
auto const keyType = newKey->type();
if (keyType <= TInt) {
gen(env, ContArUpdateIdx, fp(env), newKey);
}
yieldReturnControl(env);
}
// All accesses to the stack and locals in this function use DataTypeGeneric so
// this function should only be used for inspecting state; when the values are
// actually used they must be constrained further.
Type predictedTypeFromLocation(HTS& env, const Location& loc) {
switch (loc.space) {
case Location::Stack: {
auto i = loc.offset;
assert(i >= 0);
if (i < env.irb->evalStack().size()) {
return top(env, i, DataTypeGeneric)->type();
} else {
auto stackVal =
getStackValue(
env.irb->sp(),
i - env.irb->evalStack().size() + env.irb->stackDeficit()
);
if (stackVal.knownType.isBoxed() &&
!(stackVal.predictedInner <= Type::Bottom)) {
return ldRefReturn(stackVal.predictedInner.unbox()).box();
}
return stackVal.knownType;
}
} break;
case Location::Local:
return env.irb->predictedLocalType(loc.offset);
case Location::Litstr:
return Type::cns(curUnit(env)->lookupLitstrId(loc.offset));
case Location::Litint:
return Type::cns(loc.offset);
case Location::This:
// Don't specialize $this for cloned closures which may have been re-bound
if (curFunc(env)->hasForeignThis()) return Type::Obj;
if (auto const cls = curFunc(env)->cls()) {
return Type::Obj.specialize(cls);
}
return Type::Obj;
case Location::Iter:
case Location::Invalid:
break;
}
not_reached();
}
// All accesses to the stack and locals in this function use DataTypeGeneric so
// this function should only be used for inspecting state; when the values are
// actually used they must be constrained further.
Type predictedTypeFromLocation(IRGS& env, const Location& loc) {
switch (loc.space) {
case Location::Stack: {
auto i = loc.bcRelOffset;
assertx(i >= 0);
if (i < env.irb->evalStack().size()) {
return topType(env, i, DataTypeGeneric);
} else {
auto stackTy = env.irb->stackType(
offsetFromIRSP(env, i),
DataTypeGeneric
);
if (stackTy <= Type::BoxedCell) {
return env.irb->stackInnerTypePrediction(
offsetFromIRSP(env, i)).box();
}
return stackTy;
}
} break;
case Location::Local:
return env.irb->predictedLocalType(loc.offset);
case Location::Litstr:
return Type::cns(curUnit(env)->lookupLitstrId(loc.offset));
case Location::Litint:
return Type::cns(loc.offset);
case Location::This:
// Don't specialize $this for cloned closures which may have been re-bound
if (curFunc(env)->hasForeignThis()) return Type::Obj;
if (auto const cls = curFunc(env)->cls()) {
return Type::SubObj(cls);
}
return Type::Obj;
case Location::Iter:
case Location::Invalid:
break;
}
not_reached();
}
void CodeGenerator::cgGuardLoc(IRInstruction* inst) {
auto const rFP = x2a(m_regs[inst->src(0)].reg());
auto const baseOff = localOffset(inst->extra<GuardLoc>()->locId);
emitTypeTest(
inst->typeParam(),
rFP[baseOff + TVOFF(m_type)],
rFP[baseOff + TVOFF(m_data)],
[&] (ConditionCode cc) {
auto const destSK = SrcKey(curFunc(), m_unit.bcOff());
auto const destSR = m_tx64->getSrcRec(destSK);
destSR->emitFallbackJump(this->m_mainCode, ccNegate(cc));
});
}
void emitStaticLoc(HTS& env, int32_t locId, const StringData* name) {
if (curFunc(env)->isPseudoMain()) PUNT(StaticLoc);
auto const ldPMExit = makePseudoMainExit(env);
auto const box = curFunc(env)->isClosureBody() ?
gen(env, ClosureStaticLocInit,
cns(env, name), fp(env), cns(env, Type::Uninit)) :
gen(env, LdStaticLocCached, StaticLocName { curFunc(env), name });
auto const res = cond(
env,
0,
[&] (Block* taken) {
gen(env, CheckStaticLocInit, taken, box);
},
[&] { // Next: the static local is already initialized
return cns(env, true);
},
[&] { // Taken: need to initialize the static local
/*
* Even though this path is "cold", we're not marking it
* unlikely because the size of the instructions this will
* generate is about 10 bytes, which is not much larger than the
* 5 byte jump to acold would be.
*
* One note about StaticLoc: we're literally always going to
* generate a fallthrough trace here that is cold (the code that
* initializes the static local). TODO(#2894612).
*/
gen(env, StaticLocInitCached, box, cns(env, Type::InitNull));
return cns(env, false);
});
gen(env, IncRef, box);
auto const oldValue = ldLoc(env, locId, ldPMExit, DataTypeGeneric);
stLocRaw(env, locId, fp(env), box);
gen(env, DecRef, oldValue);
push(env, res);
}
void CodeGenerator::cgGuardStk(IRInstruction* inst) {
auto const rSP = x2a(curOpd(inst->src(0)).reg());
auto const baseOff = cellsToBytes(inst->extra<GuardStk>()->offset);
emitTypeTest(
inst->typeParam(),
rSP[baseOff + TVOFF(m_type)],
rSP[baseOff + TVOFF(m_data)],
[&] (ConditionCode cc) {
auto const destSK = SrcKey(curFunc(), m_unit.bcOff());
auto const destSR = m_tx64->getSrcRec(destSK);
destSR->emitFallbackJump(this->m_mainCode, ccNegate(cc));
});
}
void endInlinedCommon(HTS& env) {
assert(!env.fpiActiveStack.empty());
assert(!curFunc(env)->isPseudoMain());
assert(!resumed(env));
decRefLocalsInline(env);
if (curFunc(env)->mayHaveThis()) {
gen(env, DecRefThis, fp(env));
}
/*
* Pop the ActRec and restore the stack and frame pointers. It's
* important that this does endInlining before pushing the return
* value so stack offsets are properly tracked.
*/
gen(env, InlineReturn, fp(env));
// Return to the caller function. Careful between here and the
// updateMarker() below, where the caller state isn't entirely set up.
env.bcStateStack.pop_back();
env.fpiActiveStack.pop();
updateMarker(env);
gen(env, ResetSP, StackOffset{env.irb->spOffset()}, fp(env));
/*
* After the end of inlining, we are restoring to a previously defined stack
* that we know is entirely materialized (i.e. in memory), so stackDeficit
* needs to be slammed to zero.
*
* The push of the return value in the caller of this function is not yet
* materialized.
*/
assert(env.irb->evalStack().empty());
env.irb->clearStackDeficit();
FTRACE(1, "]]] end inlining: {}\n", curFunc(env)->fullName()->data());
}
void emitYield(IRGS& env) {
auto const resumeOffset = nextBcOff(env);
assertx(resumed(env));
assertx(curFunc(env)->isGenerator());
if (curFunc(env)->isAsyncGenerator()) PUNT(Yield-AsyncGenerator);
yieldImpl(env, resumeOffset);
// take a fast path if this generator has no yield k => v;
if (curFunc(env)->isPairGenerator()) {
auto const newIdx = gen(env, ContArIncIdx, fp(env));
auto const oldKey = gen(env, LdContArKey, TCell, fp(env));
gen(env, StContArKey, fp(env), newIdx);
gen(env, DecRef, oldKey);
} else {
// we're guaranteed that the key is an int
gen(env, ContArIncKey, fp(env));
}
yieldReturnControl(env);
}
void CodeGenerator::cgLdContArRaw(IRInstruction* inst) {
auto destReg = x2a(curOpd(inst->dst()).reg());
auto contArReg = x2a(curOpd(inst->src(0)).reg());
auto kind = inst->src(1)->getValInt();
auto const& slot = RawMemSlot::Get(RawMemSlot::Kind(kind));
auto off = slot.offset() - c_Continuation::getArOffset(curFunc());
switch (slot.size()) {
case sz::byte: m_as. Ldrb (destReg.W(), contArReg[off]); break;
case sz::dword: m_as. Ldr (destReg.W(), contArReg[off]); break;
case sz::qword: m_as. Ldr (destReg, contArReg[off]); break;
default: not_implemented();
}
}
void CodeGenerator::cgSideExitGuardStk(IRInstruction* inst) {
auto const sp = x2a(curOpd(inst->src(0)).reg());
auto const extra = inst->extra<SideExitGuardStk>();
emitTypeTest(
inst->typeParam(),
sp[cellsToBytes(extra->checkedSlot) + TVOFF(m_type)],
sp[cellsToBytes(extra->checkedSlot) + TVOFF(m_data)],
[&] (ConditionCode cc) {
auto const sk = SrcKey(curFunc(), extra->taken);
emitBindSideExit(this->m_mainCode, this->m_stubsCode, sk, ccNegate(cc));
}
);
}
void emitBindL(HTS& env, int32_t id) {
if (curFunc(env)->isPseudoMain()) {
interpOne(env, Type::BoxedInitCell, 1);
return;
}
auto const ldPMExit = makePseudoMainExit(env);
auto const newValue = popV(env);
// Note that the IncRef must happen first, for correctness in a
// pseudo-main: the destructor could decref the value again after
// we've stored it into the local.
pushIncRef(env, newValue);
auto const oldValue = ldLoc(env, id, ldPMExit, DataTypeSpecific);
stLocRaw(env, id, fp(env), newValue);
gen(env, DecRef, oldValue);
}
void endRegion(HTS& env, Offset nextPc) {
if (nextPc >= curFunc(env)->past()) {
// We have fallen off the end of the func's bytecodes. This happens
// when the function's bytecodes end with an unconditional
// backwards jump so that nextPc is out of bounds and causes an
// assertion failure in unit.cpp. The common case for this comes
// from the default value funclets, which are placed after the end
// of the function, with an unconditional branch back to the start
// of the function. So you should see this in any function with
// default params.
return;
}
prepareForNextHHBC(env, nullptr, nextPc, true);
auto const stack = spillStack(env);
gen(env, SyncABIRegs, fp(env), stack);
gen(env, ReqBindJmp, ReqBindJmpData(nextPc));
}
static void recordActRecPush(NormalizedInstruction& i,
const Unit* unit,
const StringData* name,
const StringData* clsName,
bool staticCall) {
const SrcKey& sk = i.source;
FTRACE(2, "annotation: recordActRecPush: {}@{} {}{}{} ({}static)\n",
unit->filepath()->data(),
sk.offset(),
clsName ? clsName->data() : "",
clsName ? "::" : "",
name,
!staticCall ? "non" : "");
SrcKey next(sk);
next.advance(unit);
const FPIEnt *fpi = curFunc()->findFPI(next.offset());
assert(fpi);
assert(name->isStatic());
assert(sk.offset() == fpi->m_fpushOff);
SrcKey fcall = sk;
fcall.m_offset = fpi->m_fcallOff;
assert(isFCallStar(*unit->at(fcall.offset())));
if (clsName) {
const Class* cls = Unit::lookupUniqueClass(clsName);
bool magic = false;
const Func* func = lookupImmutableMethod(cls, name, magic, staticCall);
if (func) {
recordFunc(i, fcall, func);
}
return;
}
const Func* func = Unit::lookupFunc(name);
if (func && func->isNameBindingImmutable(unit)) {
// this will never go into a call cache, so we dont need to
// encode the args. it will be used in OpFCall below to
// set the i->funcd.
recordFunc(i, fcall, func);
} else {
// It's not enough to remember the function name; we also need to encode
// the number of arguments and current flag disposition.
int numArgs = getImm(unit->at(sk.offset()), 0).u_IVA;
recordNameAndArgs(fcall, name, numArgs);
}
}
void CodeGenerator::cgInterpOneCommon(IRInstruction* inst) {
auto fpReg = x2a(m_regs[inst->src(0)].reg());
auto spReg = x2a(m_regs[inst->src(1)].reg());
auto pcOff = inst->extra<InterpOneData>()->bcOff;
auto opc = *(curFunc()->unit()->at(pcOff));
auto* interpOneHelper = interpOneEntryPoints[opc];
m_as. Push (x29, x30);
// TODO(2966997): this really should be saving caller-save registers and
// basically doing everything else that cgCallHelper does. This only works
// now because no caller-saved registers are live.
m_as. Mov (rHostCallReg, reinterpret_cast<uint64_t>(interpOneHelper));
m_as. Mov (argReg(0), fpReg);
m_as. Mov (argReg(1), spReg);
m_as. Mov (argReg(2), pcOff);
m_as. HostCall(3);
m_as. Pop (x30, x29);
}
/*
* Attempts to begin inlining, and returns whether or not it successed.
*
* When doing gen-time inlining, we set up a series of IR instructions
* that looks like this:
*
* fp0 = DefFP
* sp = DefSP<offset>
*
* // ... normal stuff happens ...
*
* // FPI region:
* SpillFrame sp, ...
* // ... probably some StStks due to argument expressions
* fp2 = DefInlineFP<func,retBC,retSP,off> sp
*
* // ... callee body ...
*
* InlineReturn fp2
*
* In DCE we attempt to remove the InlineReturn and DefInlineFP instructions if
* they aren't needed.
*/
bool beginInlining(IRGS& env,
unsigned numParams,
const Func* target,
Offset returnBcOffset) {
auto const& fpiStack = env.irb->fpiStack();
assertx(!fpiStack.empty() &&
"Inlining does not support calls with the FPush* in a different Tracelet");
assertx(returnBcOffset >= 0 && "returnBcOffset before beginning of caller");
assertx(curFunc(env)->base() + returnBcOffset < curFunc(env)->past() &&
"returnBcOffset past end of caller");
FTRACE(1, "[[[ begin inlining: {}\n", target->fullName()->data());
SSATmp** params = (SSATmp**)alloca(sizeof(SSATmp*) * numParams);
for (unsigned i = 0; i < numParams; ++i) {
params[numParams - i - 1] = popF(env);
}
auto const prevSP = fpiStack.front().returnSP;
auto const prevSPOff = fpiStack.front().returnSPOff;
spillStack(env);
auto const calleeSP = sp(env);
always_assert_flog(
prevSP == calleeSP,
"FPI stack pointer and callee stack pointer didn't match in beginInlining"
);
auto const& info = fpiStack.front();
always_assert(!isFPushCuf(info.fpushOpc) && !info.interp);
auto ctx = [&] {
if (info.ctx || isFPushFunc(info.fpushOpc)) {
return info.ctx;
}
constexpr int32_t adjust = offsetof(ActRec, m_r) - offsetof(ActRec, m_this);
IRSPOffset ctxOff{invSPOff(env) - info.returnSPOff - adjust};
return gen(env, LdStk, TCtx, IRSPOffsetData{ctxOff}, sp(env));
}();
DefInlineFPData data;
data.target = target;
data.retBCOff = returnBcOffset;
data.fromFPushCtor = isFPushCtor(info.fpushOpc);
data.ctx = ctx;
data.retSPOff = prevSPOff;
data.spOffset = offsetFromIRSP(env, BCSPOffset{0});
// Push state and update the marker before emitting any instructions so
// they're all given markers in the callee.
auto const key = SrcKey {
target,
target->getEntryForNumArgs(numParams),
false
};
env.bcStateStack.emplace_back(key);
env.inlineLevel++;
updateMarker(env);
auto const calleeFP = gen(env, DefInlineFP, data, calleeSP, fp(env));
for (unsigned i = 0; i < numParams; ++i) {
stLocRaw(env, i, calleeFP, params[i]);
}
const bool hasVariadicArg = target->hasVariadicCaptureParam();
for (unsigned i = numParams; i < target->numLocals() - hasVariadicArg; ++i) {
/*
* Here we need to be generating hopefully-dead stores to initialize
* non-parameter locals to KindOfUninit in case we have to leave the trace.
*/
stLocRaw(env, i, calleeFP, cns(env, TUninit));
}
if (hasVariadicArg) {
auto argNum = target->numLocals() - 1;
always_assert(numParams <= argNum);
stLocRaw(env, argNum, calleeFP, cns(env, staticEmptyArray()));
}
return true;
}
Block* makePseudoMainExit(IRGS& env) {
return curFunc(env)->isPseudoMain()
? makeExit(env)
: nullptr;
}
bool RegionFormer::tryInline(uint32_t& instrSize) {
assertx(m_inst.source == m_sk);
assertx(m_inst.func() == curFunc());
assertx(m_sk.resumed() == resumed());
instrSize = 0;
if (!m_inl.canInlineAt(m_inst.source, m_inst.funcd, *m_region)) {
return false;
}
auto refuse = [this](const std::string& str) {
FTRACE(2, "selectTracelet not inlining {}: {}\n",
m_inst.toString(), str);
return false;
};
auto callee = m_inst.funcd;
// Make sure the FPushOp wasn't interpreted.
if (m_irgs.fpiStack.empty()) {
return refuse("fpistack empty; fpush was in a different region");
}
auto spillFrame = m_irgs.fpiStack.top().spillFrame;
if (!spillFrame) {
return refuse("couldn't find SpillFrame for FPushOp");
}
auto numArgs = m_inst.imm[0].u_IVA;
auto numParams = callee->numParams();
// Set up the region context, mapping stack slots in the caller to locals in
// the callee.
RegionContext ctx;
ctx.func = callee;
ctx.bcOffset = callee->getEntryForNumArgs(numArgs);
ctx.spOffset = FPInvOffset{safe_cast<int32_t>(callee->numSlotsInFrame())};
ctx.resumed = false;
for (int i = 0; i < numArgs; ++i) {
auto type = irgen::publicTopType(m_irgs, BCSPOffset{i});
uint32_t paramIdx = numArgs - 1 - i;
ctx.liveTypes.push_back({RegionDesc::Location::Local{paramIdx}, type});
}
for (unsigned i = numArgs; i < numParams; ++i) {
// These locals will be populated by DV init funclets but they'll start
// out as Uninit.
ctx.liveTypes.push_back({RegionDesc::Location::Local{i}, TUninit});
}
FTRACE(1, "selectTracelet analyzing callee {} with context:\n{}",
callee->fullName()->data(), show(ctx));
auto region = selectTracelet(ctx, m_profiling, false /* noinline */);
if (!region) {
return refuse("failed to select region in callee");
}
instrSize = region->instrSize();
auto newInstrSize = instrSize + m_numBCInstrs + m_pendingInlinedInstrs;
if (newInstrSize > RuntimeOption::EvalJitMaxRegionInstrs) {
return refuse("new region would be too large");
}
if (!m_inl.shouldInline(callee, *region)) {
return refuse("shouldIRInline failed");
}
return true;
}
